diff --git a/docs/versioned-plugins/codecs/cef-index.asciidoc b/docs/versioned-plugins/codecs/cef-index.asciidoc index be5ec1776..9ff439b2c 100644 --- a/docs/versioned-plugins/codecs/cef-index.asciidoc +++ b/docs/versioned-plugins/codecs/cef-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-10-22 | <> | 2023-05-12 | <> | 2022-10-26 | <> | 2022-04-20 @@ -29,6 +30,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::cef-v6.2.8.asciidoc[] include::cef-v6.2.7.asciidoc[] include::cef-v6.2.6.asciidoc[] include::cef-v6.2.5.asciidoc[] diff --git a/docs/versioned-plugins/codecs/cef-v6.2.8.asciidoc b/docs/versioned-plugins/codecs/cef-v6.2.8.asciidoc new file mode 100644 index 000000000..3470a5af0 --- /dev/null +++ b/docs/versioned-plugins/codecs/cef-v6.2.8.asciidoc @@ -0,0 +1,602 @@ +:plugin: cef +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.2.8 +:release_date: 2024-10-22 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-cef/blob/v6.2.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cef codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Implementation of a Logstash codec for the ArcSight Common Event Format (CEF). +It is based on https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors/pdfdoc/common-event-format-v25/common-event-format-v25.pdf[Implementing ArcSight CEF Revision 25, September 2017]. + +If this codec receives a payload from an input that is not a valid CEF message, then it +produces an event with the payload as the 'message' field and a '_cefparsefailure' tag. + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin can be used to decode CEF events _into_ the Elastic Common Schema, or to encode ECS-compatible events into CEF. +It can also be used _without_ ECS, encoding and decoding events using only CEF-defined field names and keys. + +The ECS Compatibility mode for a specific plugin instance can be controlled by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> when defining the codec: + +[source,sh] +----- + input { + tcp { + # ... + codec => cef { + ecs_compatibility => v1 + } + } + } +----- + +If left unspecified, the value of the `pipeline.ecs_compatibility` setting is used. + +===== Timestamps and ECS compatiblity + +When decoding in ECS Compatibility Mode, timestamp-type fields are parsed and normalized +to specific points on the timeline. + +Because the CEF format allows ambiguous timestamp formats, some reasonable assumptions are made: + + - When the timestamp does not include a year, we assume it happened in the recent past + (or _very_ near future to accommodate out-of-sync clocks and timezone offsets). + - When the timestamp does not include UTC-offset information, we use the event's + timezone (`dtz` or `deviceTimeZone` field), or fall through to this plugin's + <<{version}-plugins-{type}s-{plugin}-default_timezone>>. + - Localized timestamps are parsed using the provided <<{version}-plugins-{type}s-{plugin}-locale>>. + +[id="{version}-plugins-{type}s-{plugin}-field-mapping"] +===== Field mapping + +The header fields from each CEF payload is expanded to the following fields, depending on whether ECS is enabled. + +[id="{version}-plugins-{type}s-{plugin}-header-field"] +===== Header field mapping +|===== +|ECS Disabled | ECS Field + +|`cefVersion` |`[cef][version]` +|`deviceVendor` |`[observer][vendor]` +|`deviceProduct` |`[observer][product]` +|`deviceVersion` |`[observer][version]` +|`deviceEventClassId`|`[event][code]` +|`name` |`[cef][name]` +|`severity` |`[event][severity]` +|===== + +When decoding CEF payloads with `ecs_compatibility => disabled`, the abbreviated CEF Keys found in extensions are expanded, and CEF Field Names are inserted at the root level of the event. + +When decoding in an ECS Compatibility mode, the ECS Fields are populated from the corresponding CEF Field Names _or_ CEF Keys found in the payload's extensions. + +The following is a mapping between these fields. + +// Templates for short-hand notes in the table below +:cef-ambiguous-higher: pass:quotes[Multiple possible CEF fields map to this ECS Field. When decoding, the last entry encountered wins. When encoding, this field has _higher_ priority.] +:cef-ambiguous-lower: pass:quotes[Multiple possible CEF fields map to this ECS Field. When decoding, the last entry encountered wins. When encoding, this field has _lower_ priority.] +:cef-normalize-timestamp: pass:quotes[This field contains a timestamp. In ECS Compatibility Mode, it is parsed to a specific point in time.] +:cef-plugin-config-condition: pass:quotes[When plugin configured with] + + +[id="{version}-plugins-{type}s-{plugin}-ext-field"] +===== Extension field mapping +|======================================================================================================================= +|CEF Field Name (optional CEF Key) |ECS Field + +|`agentAddress` (`agt`) |`[agent][ip]` +|`agentDnsDomain` |`[cef][agent][registered_domain]` + + {cef-ambiguous-higher} +|`agentHostName` (`ahost`) |`[agent][name]` +|`agentId` (`aid`) |`[agent][id]` +|`agentMacAddress` (`amac`) |`[agent][mac]` +|`agentNtDomain` |`[cef][agent][registered_domain]` + + {cef-ambiguous-lower} +|`agentReceiptTime` (`art`) |`[event][created]` + + {cef-normalize-timestamp} +|`agentTimeZone` (`atz`) |`[cef][agent][timezone]` +|`agentTranslatedAddress` |`[cef][agent][nat][ip]` +|`agentTranslatedZoneExternalID` |`[cef][agent][translated_zone][external_id]` +|`agentTranslatedZoneURI` |`[cef][agent][translated_zone][uri]` +|`agentType` (`at`) |`[agent][type]` +|`agentVersion` (`av`) |`[agent][version]` +|`agentZoneExternalID` |`[cef][agent][zone][external_id]` +|`agentZoneURI` |`[cef][agent][zone][uri]` +|`applicationProtocol` (`app`) |`[network][protocol]` +|`baseEventCount` (`cnt`) |`[cef][base_event_count]` +|`bytesIn` (`in`) |`[source][bytes]` +|`bytesOut` (`out`) |`[destination][bytes]` +|`categoryDeviceType` (`catdt`) |`[cef][device_type]` +|`customerExternalID` |`[organization][id]` +|`customerURI` |`[organization][name]` +|`destinationAddress` (`dst`) |`[destination][ip]` +|`destinationDnsDomain` |`[destination][registered_domain]` + + {cef-ambiguous-higher} +|`destinationGeoLatitude` (`dlat`) |`[destination][geo][location][lat]` +|`destinationGeoLongitude` (`dlong`) |`[destination][geo][location][lon]` +|`destinationHostName` (`dhost`) |`[destination][domain]` +|`destinationMacAddress` (`dmac`) |`[destination][mac]` +|`destinationNtDomain` (`dntdom`) |`[destination][registered_domain]` + + {cef-ambiguous-lower} +|`destinationPort` (`dpt`) |`[destination][port]` +|`destinationProcessId` (`dpid`) |`[destination][process][pid]` +|`destinationProcessName` (`dproc`) |`[destination][process][name]` +|`destinationServiceName` |`[destination][service][name]` +|`destinationTranslatedAddress` |`[destination][nat][ip]` +|`destinationTranslatedPort` |`[destination][nat][port]` +|`destinationTranslatedZoneExternalID` |`[cef][destination][translated_zone][external_id]` +|`destinationTranslatedZoneURI` |`[cef][destination][translated_zone][uri]` +|`destinationUserId` (`duid`) |`[destination][user][id]` +|`destinationUserName` (`duser`) |`[destination][user][name]` +|`destinationUserPrivileges` (`dpriv`) |`[destination][user][group][name]` +|`destinationZoneExternalID` |`[cef][destination][zone][external_id]` +|`destinationZoneURI` |`[cef][destination][zone][uri]` +|`deviceAction` (`act`) |`[event][action]` +.2+|`deviceAddress` (`dvc`) |`[observer][ip]` + + {cef-plugin-config-condition} `device => observer` + |`[host][ip]` + + {cef-plugin-config-condition} `device => host` +|`deviceCustomFloatingPoint1` (`cfp1`) |`[cef][device_custom_floating_point_1][value]` +|`deviceCustomFloatingPoint1Label` (`cfp1Label`)|`[cef][device_custom_floating_point_1][label]` +|`deviceCustomFloatingPoint2` (`cfp2`) |`[cef][device_custom_floating_point_2][value]` +|`deviceCustomFloatingPoint2Label` (`cfp2Label`)|`[cef][device_custom_floating_point_2][label]` +|`deviceCustomFloatingPoint3` (`cfp3`) |`[cef][device_custom_floating_point_3][value]` +|`deviceCustomFloatingPoint3Label` (`cfp3Label`)|`[cef][device_custom_floating_point_3][label]` +|`deviceCustomFloatingPoint4` (`cfp4`) |`[cef][device_custom_floating_point_4][value]` +|`deviceCustomFloatingPoint4Label` (`cfp4Label`)|`[cef][device_custom_floating_point_4][label]` +|`deviceCustomFloatingPoint5` (`cfp5`) |`[cef][device_custom_floating_point_5][value]` +|`deviceCustomFloatingPoint5Label` (`cfp5Label`)|`[cef][device_custom_floating_point_5][label]` +|`deviceCustomFloatingPoint6` (`cfp6`) |`[cef][device_custom_floating_point_6][value]` +|`deviceCustomFloatingPoint6Label` (`cfp6Label`)|`[cef][device_custom_floating_point_6][label]` +|`deviceCustomFloatingPoint7` (`cfp7`) |`[cef][device_custom_floating_point_7][value]` +|`deviceCustomFloatingPoint7Label` (`cfp7Label`)|`[cef][device_custom_floating_point_7][label]` +|`deviceCustomFloatingPoint8` (`cfp8`) |`[cef][device_custom_floating_point_8][value]` +|`deviceCustomFloatingPoint8Label` (`cfp8Label`)|`[cef][device_custom_floating_point_8][label]` +|`deviceCustomFloatingPoint9` (`cfp9`) |`[cef][device_custom_floating_point_9][value]` +|`deviceCustomFloatingPoint9Label` (`cfp9Label`)|`[cef][device_custom_floating_point_9][label]` +|`deviceCustomFloatingPoint10` (`cfp10`) |`[cef][device_custom_floating_point_10][value]` +|`deviceCustomFloatingPoint10Label` (`cfp10Label`)|`[cef][device_custom_floating_point_10][label]` +|`deviceCustomFloatingPoint11` (`cfp11`) |`[cef][device_custom_floating_point_11][value]` +|`deviceCustomFloatingPoint11Label` (`cfp11Label`)|`[cef][device_custom_floating_point_11][label]` +|`deviceCustomFloatingPoint12` (`cfp12`) |`[cef][device_custom_floating_point_12][value]` +|`deviceCustomFloatingPoint12Label` (`cfp12Label`)|`[cef][device_custom_floating_point_12][label]` +|`deviceCustomFloatingPoint13` (`cfp13`) |`[cef][device_custom_floating_point_13][value]` +|`deviceCustomFloatingPoint13Label` (`cfp13Label`)|`[cef][device_custom_floating_point_13][label]` +|`deviceCustomFloatingPoint14` (`cfp14`) |`[cef][device_custom_floating_point_14][value]` +|`deviceCustomFloatingPoint14Label` (`cfp14Label`)|`[cef][device_custom_floating_point_14][label]` +|`deviceCustomFloatingPoint15` (`cfp15`) |`[cef][device_custom_floating_point_15][value]` +|`deviceCustomFloatingPoint15Label` (`cfp15Label`)|`[cef][device_custom_floating_point_15][label]` +|`deviceCustomIPv6Address1` (`c6a1`) |`[cef][device_custom_ipv6_address_1][value]` +|`deviceCustomIPv6Address1Label` (`c6a1Label`) |`[cef][device_custom_ipv6_address_1][label]` +|`deviceCustomIPv6Address2` (`c6a2`) |`[cef][device_custom_ipv6_address_2][value]` +|`deviceCustomIPv6Address2Label` (`c6a2Label`) |`[cef][device_custom_ipv6_address_2][label]` +|`deviceCustomIPv6Address3` (`c6a3`) |`[cef][device_custom_ipv6_address_3][value]` +|`deviceCustomIPv6Address3Label` (`c6a3Label`) |`[cef][device_custom_ipv6_address_3][label]` +|`deviceCustomIPv6Address4` (`c6a4`) |`[cef][device_custom_ipv6_address_4][value]` +|`deviceCustomIPv6Address4Label` (`c6a4Label`) |`[cef][device_custom_ipv6_address_4][label]` +|`deviceCustomIPv6Address5` (`c6a5`) |`[cef][device_custom_ipv6_address_5][value]` +|`deviceCustomIPv6Address5Label` (`c6a5Label`) |`[cef][device_custom_ipv6_address_5][label]` +|`deviceCustomIPv6Address6` (`c6a6`) |`[cef][device_custom_ipv6_address_6][value]` +|`deviceCustomIPv6Address6Label` (`c6a6Label`) |`[cef][device_custom_ipv6_address_6][label]` +|`deviceCustomIPv6Address7` (`c6a7`) |`[cef][device_custom_ipv6_address_7][value]` +|`deviceCustomIPv6Address7Label` (`c6a7Label`) |`[cef][device_custom_ipv6_address_7][label]` +|`deviceCustomIPv6Address8` (`c6a8`) |`[cef][device_custom_ipv6_address_8][value]` +|`deviceCustomIPv6Address8Label` (`c6a8Label`) |`[cef][device_custom_ipv6_address_8][label]` +|`deviceCustomIPv6Address9` (`c6a9`) |`[cef][device_custom_ipv6_address_9][value]` +|`deviceCustomIPv6Address9Label` (`c6a9Label`) |`[cef][device_custom_ipv6_address_9][label]` +|`deviceCustomIPv6Address10` (`c6a10`) |`[cef][device_custom_ipv6_address_10][value]` +|`deviceCustomIPv6Address10Label` (`c6a10Label`)|`[cef][device_custom_ipv6_address_10][label]` +|`deviceCustomIPv6Address11` (`c6a11`) |`[cef][device_custom_ipv6_address_11][value]` +|`deviceCustomIPv6Address11Label` (`c6a11Label`)|`[cef][device_custom_ipv6_address_11][label]` +|`deviceCustomIPv6Address12` (`c6a12`) |`[cef][device_custom_ipv6_address_12][value]` +|`deviceCustomIPv6Address12Label` (`c6a12Label`)|`[cef][device_custom_ipv6_address_12][label]` +|`deviceCustomIPv6Address13` (`c6a13`) |`[cef][device_custom_ipv6_address_13][value]` +|`deviceCustomIPv6Address13Label` (`c6a13Label`)|`[cef][device_custom_ipv6_address_13][label]` +|`deviceCustomIPv6Address14` (`c6a14`) |`[cef][device_custom_ipv6_address_14][value]` +|`deviceCustomIPv6Address14Label` (`c6a14Label`)|`[cef][device_custom_ipv6_address_14][label]` +|`deviceCustomIPv6Address15` (`c6a15`) |`[cef][device_custom_ipv6_address_15][value]` +|`deviceCustomIPv6Address15Label` (`c6a15Label`)|`[cef][device_custom_ipv6_address_15][label]` +|`deviceCustomNumber1` (`cn1`) |`[cef][device_custom_number_1][value]` +|`deviceCustomNumber1Label` (`cn1Label`) |`[cef][device_custom_number_1][label]` +|`deviceCustomNumber2` (`cn2`) |`[cef][device_custom_number_2][value]` +|`deviceCustomNumber2Label` (`cn2Label`) |`[cef][device_custom_number_2][label]` +|`deviceCustomNumber3` (`cn3`) |`[cef][device_custom_number_3][value]` +|`deviceCustomNumber3Label` (`cn3Label`) |`[cef][device_custom_number_3][label]` +|`deviceCustomNumber4` (`cn4`) |`[cef][device_custom_number_4][value]` +|`deviceCustomNumber4Label` (`cn4Label`) |`[cef][device_custom_number_4][label]` +|`deviceCustomNumber5` (`cn5`) |`[cef][device_custom_number_5][value]` +|`deviceCustomNumber5Label` (`cn5Label`) |`[cef][device_custom_number_5][label]` +|`deviceCustomNumber6` (`cn6`) |`[cef][device_custom_number_6][value]` +|`deviceCustomNumber6Label` (`cn6Label`) |`[cef][device_custom_number_6][label]` +|`deviceCustomNumber7` (`cn7`) |`[cef][device_custom_number_7][value]` +|`deviceCustomNumber7Label` (`cn7Label`) |`[cef][device_custom_number_7][label]` +|`deviceCustomNumber8` (`cn8`) |`[cef][device_custom_number_8][value]` +|`deviceCustomNumber8Label` (`cn8Label`) |`[cef][device_custom_number_8][label]` +|`deviceCustomNumber9` (`cn9`) |`[cef][device_custom_number_9][value]` +|`deviceCustomNumber9Label` (`cn9Label`) |`[cef][device_custom_number_9][label]` +|`deviceCustomNumber10` (`cn10`) |`[cef][device_custom_number_10][value]` +|`deviceCustomNumber10Label` (`cn10Label`) |`[cef][device_custom_number_10][label]` +|`deviceCustomNumber11` (`cn11`) |`[cef][device_custom_number_11][value]` +|`deviceCustomNumber11Label` (`cn11Label`) |`[cef][device_custom_number_11][label]` +|`deviceCustomNumber12` (`cn12`) |`[cef][device_custom_number_12][value]` +|`deviceCustomNumber12Label` (`cn12Label`) |`[cef][device_custom_number_12][label]` +|`deviceCustomNumber13` (`cn13`) |`[cef][device_custom_number_13][value]` +|`deviceCustomNumber13Label` (`cn13Label`) |`[cef][device_custom_number_13][label]` +|`deviceCustomNumber14` (`cn14`) |`[cef][device_custom_number_14][value]` +|`deviceCustomNumber14Label` (`cn14Label`) |`[cef][device_custom_number_14][label]` +|`deviceCustomNumber15` (`cn15`) |`[cef][device_custom_number_15][value]` +|`deviceCustomNumber15Label` (`cn15Label`) |`[cef][device_custom_number_15][label]` +|`deviceCustomString1` (`cs1`) |`[cef][device_custom_string_1][value]` +|`deviceCustomString1Label` (`cs1Label`) |`[cef][device_custom_string_1][label]` +|`deviceCustomString2` (`cs2`) |`[cef][device_custom_string_2][value]` +|`deviceCustomString2Label` (`cs2Label`) |`[cef][device_custom_string_2][label]` +|`deviceCustomString3` (`cs3`) |`[cef][device_custom_string_3][value]` +|`deviceCustomString3Label` (`cs3Label`) |`[cef][device_custom_string_3][label]` +|`deviceCustomString4` (`cs4`) |`[cef][device_custom_string_4][value]` +|`deviceCustomString4Label` (`cs4Label`) |`[cef][device_custom_string_4][label]` +|`deviceCustomString5` (`cs5`) |`[cef][device_custom_string_5][value]` +|`deviceCustomString5Label` (`cs5Label`) |`[cef][device_custom_string_5][label]` +|`deviceCustomString6` (`cs6`) |`[cef][device_custom_string_6][value]` +|`deviceCustomString6Label` (`cs6Label`) |`[cef][device_custom_string_6][label]` +|`deviceCustomString7` (`cs7`) |`[cef][device_custom_string_7][value]` +|`deviceCustomString7Label` (`cs7Label`) |`[cef][device_custom_string_7][label]` +|`deviceCustomString8` (`cs8`) |`[cef][device_custom_string_8][value]` +|`deviceCustomString8Label` (`cs8Label`) |`[cef][device_custom_string_8][label]` +|`deviceCustomString9` (`cs9`) |`[cef][device_custom_string_9][value]` +|`deviceCustomString9Label` (`cs9Label`) |`[cef][device_custom_string_9][label]` +|`deviceCustomString10` (`cs10`) |`[cef][device_custom_string_10][value]` +|`deviceCustomString10Label` (`cs10Label`) |`[cef][device_custom_string_10][label]` +|`deviceCustomString11` (`cs11`) |`[cef][device_custom_string_11][value]` +|`deviceCustomString11Label` (`cs11Label`) |`[cef][device_custom_string_11][label]` +|`deviceCustomString12` (`cs12`) |`[cef][device_custom_string_12][value]` +|`deviceCustomString12Label` (`cs12Label`) |`[cef][device_custom_string_12][label]` +|`deviceCustomString13` (`cs13`) |`[cef][device_custom_string_13][value]` +|`deviceCustomString13Label` (`cs13Label`) |`[cef][device_custom_string_13][label]` +|`deviceCustomString14` (`cs14`) |`[cef][device_custom_string_14][value]` +|`deviceCustomString14Label` (`cs14Label`) |`[cef][device_custom_string_14][label]` +|`deviceCustomString15` (`cs15`) |`[cef][device_custom_string_15][value]` +|`deviceCustomString15Label` (`cs15Label`) |`[cef][device_custom_string_15][label]` +|`deviceDirection` |`[network][direction]` +.2+|`deviceDnsDomain` |`[observer][registered_domain]` + + {cef-plugin-config-condition} `device => observer`. + |`[host][registered_domain]` + + {cef-plugin-config-condition} `device => host`. +|`deviceEventCategory` (`cat`) |`[cef][category]` +.2+|`deviceExternalId` |`[observer][name]` + + {cef-plugin-config-condition} `device => observer`. + |`[host][id]` + + {cef-plugin-config-condition} `device => host`. +|`deviceFacility` |`[log][syslog][facility][code]` +.2+|`deviceHostName` (`dvchost`) |`[observer][hostname]` + + {cef-plugin-config-condition} `device => observer`. + |`[host][name]` + + {cef-plugin-config-condition} `device => host`. +|`deviceInboundInterface` |`[observer][ingress][interface][name]` +.2+|`deviceMacAddress` (`dvcmac`) |`[observer][mac]` + + {cef-plugin-config-condition} `device => observer`. + |`[host][mac]` + + {cef-plugin-config-condition} `device => host`. +|`deviceNtDomain` |`[cef][nt_domain]` +|`deviceOutboundInterface` |`[observer][egress][interface][name]` +|`devicePayloadId` |`[cef][payload_id]` +|`deviceProcessId` (`dvcpid`) |`[process][pid]` +|`deviceProcessName` |`[process][name]` +|`deviceReceiptTime` (`rt`) |`@timestamp` + + {cef-normalize-timestamp} +|`deviceTimeZone` (`dtz`) |`[event][timezone]` +|`deviceTranslatedAddress` |`[host][nat][ip]` +|`deviceTranslatedZoneExternalID` |`[cef][translated_zone][external_id]` +|`deviceTranslatedZoneURI` |`[cef][translated_zone][uri]` +|`deviceVersion` |`[observer][version]` +|`deviceZoneExternalID` |`[cef][zone][external_id]` +|`deviceZoneURI` |`[cef][zone][uri]` +|`endTime` (`end`) |`[event][end]` + + {cef-normalize-timestamp} +|`eventId` |`[event][id]` +|`eventOutcome` (`outcome`) |`[event][outcome]` +|`externalId` |`[cef][external_id]` +|`fileCreateTime` |`[file][created]` +|`fileHash` |`[file][hash]` +|`fileId` |`[file][inode]` +|`fileModificationTime` |`[file][mtime]` + + {cef-normalize-timestamp} +|`fileName` (`fname`) |`[file][name]` +|`filePath` |`[file][path]` +|`filePermission` |`[file][group]` +|`fileSize` (`fsize`) |`[file][size]` +|`fileType` |`[file][extension]` +|`managerReceiptTime` (`mrt`) |`[event][ingested]` + + {cef-normalize-timestamp} +|`message` (`msg`) |`[message]` +|`oldFileCreateTime` |`[cef][old_file][created]` + + {cef-normalize-timestamp} +|`oldFileHash` |`[cef][old_file][hash]` +|`oldFileId` |`[cef][old_file][inode]` +|`oldFileModificationTime` |`[cef][old_file][mtime]` + + {cef-normalize-timestamp} +|`oldFileName` |`[cef][old_file][name]` +|`oldFilePath` |`[cef][old_file][path]` +|`oldFilePermission` |`[cef][old_file][group]` +|`oldFileSize` |`[cef][old_file][size]` +|`oldFileType` |`[cef][old_file][extension]` +|`rawEvent` |`[event][original]` +|`Reason` (`reason`) |`[event][reason]` +|`requestClientApplication` |`[user_agent][original]` +|`requestContext` |`[http][request][referrer]` +|`requestCookies` |`[cef][request][cookies]` +|`requestMethod` |`[http][request][method]` +|`requestUrl` (`request`) |`[url][original]` +|`sourceAddress` (`src`) |`[source][ip]` +|`sourceDnsDomain` |`[source][registered_domain]` + + {cef-ambiguous-higher} +|`sourceGeoLatitude` (`slat`) |`[source][geo][location][lat]` +|`sourceGeoLongitude` (`slong`) |`[source][geo][location][lon]` +|`sourceHostName` (`shost`) |`[source][domain]` +|`sourceMacAddress` (`smac`) |`[source][mac]` +|`sourceNtDomain` (`sntdom`) |`[source][registered_domain]` + + {cef-ambiguous-lower} +|`sourcePort` (`spt`) |`[source][port]` +|`sourceProcessId` (`spid`) |`[source][process][pid]` +|`sourceProcessName` (`sproc`) |`[source][process][name]` +|`sourceServiceName` |`[source][service][name]` +|`sourceTranslatedAddress` |`[source][nat][ip]` +|`sourceTranslatedPort` |`[source][nat][port]` +|`sourceTranslatedZoneExternalID` |`[cef][source][translated_zone][external_id]` +|`sourceTranslatedZoneURI` |`[cef][source][translated_zone][uri]` +|`sourceUserId` (`suid`) |`[source][user][id]` +|`sourceUserName` (`suser`) |`[source][user][name]` +|`sourceUserPrivileges` (`spriv`) |`[source][user][group][name]` +|`sourceZoneExternalID` |`[cef][source][zone][external_id]` +|`sourceZoneURI` |`[cef][source][zone][uri]` +|`startTime` (`start`) |`[event][start]` + + {cef-normalize-timestamp} +|`transportProtocol` (`proto`) |`[network][transport]` +|`type` |`[cef][type]` +|======================================================================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cef Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-delimiter>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-device>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-locale>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-product>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-raw_data_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-reverse_mapping>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-severity>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-signature>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-vendor>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-default_timezone"] +===== `default_timezone` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** https://en.wikipedia.org/wiki/List_of_tz_database_time_zones[Timezone names] (such as `Europe/Moscow`, `America/Argentina/Buenos_Aires`) +** UTC Offsets (such as `-08:00`, `+03:00`) +* The default value is your system time zone +* This option has no effect when _encoding_. + +When parsing timestamp fields in ECS mode and encountering timestamps that +do not contain UTC-offset information, the `deviceTimeZone` (`dtz`) field +from the CEF payload is used to interpret the given time. If the event does +not include timezone information, this `default_timezone` is used instead. + +[id="{version}-plugins-{type}s-{plugin}-delimiter"] +===== `delimiter` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If your input puts a delimiter between each CEF event, you'll want to set +this to be that delimiter. + +NOTE: Byte stream inputs such as TCP require delimiter to be specified. Otherwise input can be truncated or incorrectly split. + +**Example** + +[source,ruby] +----- + input { + tcp { + codec => cef { delimiter => "\r\n" } + # ... + } + } +----- + +This setting allows the following character sequences to have special meaning: + +* `\\r` (backslash "r") - means carriage return (ASCII 0x0D) +* `\\n` (backslash "n") - means newline (ASCII 0x0A) + +[id="{version}-plugins-{type}s-{plugin}-device"] +===== `device` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `observer`: indicates that device-specific fields represent the device used to _observe_ the event. +** `host`: indicates that device-specific fields represent the device on which the event _occurred_. +* The default value for this setting is `observer`. +* Option has no effect when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,`ecs_compatibility => disabled`>>. +* Option has no effect when _encoding_ + +Defines a set of device-specific CEF fields as either representing the device on which an +event _occurred_, or merely the device from which the event was _observed_. +This causes the relevant fields to be routed to either the `host` or the `observer` +top-level groupings. + +If the codec handles data from a variety of sources, the ECS recommendation is to use `observer`. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: uses CEF-defined field names in the event (e.g., `bytesIn`, `sourceAddress`) +** `v1`: supports ECS-compatible event fields (e.g., `[source][bytes]`, `[source][ip]`) +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + * Option has no effect when _decoding_ + +When this codec is used in an Output Plugin, a list of fields can be provided to be included in CEF extensions part as key/value pairs. + +[id="{version}-plugins-{type}s-{plugin}-locale"] +===== `locale` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** Abbreviated language_COUNTRY format (e.g., `en_GB`, `pt_BR`) +** Valid https://tools.ietf.org/html/bcp47[IETF BCP 47] language tag (e.g., `zh-cmn-Hans-CN`) +* The default value is your system locale +* Option has no effect when _encoding_ + +When parsing timestamp fields in ECS mode and encountering timestamps in +a localized format, this `locale` is used to interpret locale-specific strings +such as month abbreviations. + +[id="{version}-plugins-{type}s-{plugin}-name"] +===== `name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Logstash"` + * Option has no effect when _decoding_ + +When this codec is used in an Output Plugin, this option can be used to specify the +value of the name field in the CEF header. The new value can include `%{foo}` strings +to help you build a new value from other parts of the event. + +[id="{version}-plugins-{type}s-{plugin}-product"] +===== `product` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Logstash"` + * Option has no effect when _decoding_ + +When this codec is used in an Output Plugin, this option can be used to specify the +value of the device product field in CEF header. The new value can include `%{foo}` strings +to help you build a new value from other parts of the event. + +[id="{version}-plugins-{type}s-{plugin}-raw_data_field"] +===== `raw_data_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +Store the raw data to the field, for example `[event][original]`. Existing target field will be overriden. + +[id="{version}-plugins-{type}s-{plugin}-reverse_mapping"] +===== `reverse_mapping` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + * Option has no effect when _decoding_ + +Set to true to adhere to the specifications and encode using the CEF key name (short name) for the CEF field names. + +[id="{version}-plugins-{type}s-{plugin}-severity"] +===== `severity` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"6"` + * Option has no effect when _decoding_ + +When this codec is used in an Output Plugin, this option can be used to specify the +value of the severity field in CEF header. The new value can include `%{foo}` strings +to help you build a new value from other parts of the event. + +Defined as field of type string to allow sprintf. The value will be validated +to be an integer in the range from 0 to 10 (including). +All invalid values will be mapped to the default of 6. + +[id="{version}-plugins-{type}s-{plugin}-signature"] +===== `signature` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Logstash"` + * Option has no effect when _decoding_ + +When this codec is used in an Output Plugin, this option can be used to specify the +value of the signature ID field in CEF header. The new value can include `%{foo}` strings +to help you build a new value from other parts of the event. + +[id="{version}-plugins-{type}s-{plugin}-vendor"] +===== `vendor` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Elasticsearch"` + * Option has no effect when _decoding_ + +When this codec is used in an Output Plugin, this option can be used to specify the +value of the device vendor field in CEF header. The new value can include `%{foo}` strings +to help you build a new value from other parts of the event. + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1.0"` + * Option has no effect when _decoding_ + +When this codec is used in an Output Plugin, this option can be used to specify the +value of the device version field in CEF header. The new value can include `%{foo}` strings +to help you build a new value from other parts of the event. diff --git a/docs/versioned-plugins/codecs/cloudfront-index.asciidoc b/docs/versioned-plugins/codecs/cloudfront-index.asciidoc index 68d296458..1c19b37cb 100644 --- a/docs/versioned-plugins/codecs/cloudfront-index.asciidoc +++ b/docs/versioned-plugins/codecs/cloudfront-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -19,6 +23,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::cloudfront-v7.2.1.asciidoc[] +include::cloudfront-v7.2.0.asciidoc[] +include::cloudfront-v7.1.8.asciidoc[] +include::cloudfront-v7.1.7.asciidoc[] include::cloudfront-v7.1.6.asciidoc[] include::cloudfront-v7.1.5.asciidoc[] include::cloudfront-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/codecs/cloudfront-v7.1.7.asciidoc b/docs/versioned-plugins/codecs/cloudfront-v7.1.7.asciidoc new file mode 100644 index 000000000..715a61ca4 --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudfront-v7.1.7.asciidoc @@ -0,0 +1,53 @@ +:integration: aws +:plugin: cloudfront +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudfront codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This codec will read cloudfront encoded content + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudfront Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this codec. Examples include "UTF-8" and +"CP1252" + +JSON requires valid UTF-8 strings, but in some cases, software that +emits JSON does so in another encoding (nxlog, for example). In +weird cases like this, you can set the charset setting to the +actual encoding of the text and logstash will convert it for you. + +For nxlog users, you'll want to set this to "CP1252" + + diff --git a/docs/versioned-plugins/codecs/cloudfront-v7.1.8.asciidoc b/docs/versioned-plugins/codecs/cloudfront-v7.1.8.asciidoc new file mode 100644 index 000000000..cdf5136f6 --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudfront-v7.1.8.asciidoc @@ -0,0 +1,53 @@ +:integration: aws +:plugin: cloudfront +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudfront codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This codec will read cloudfront encoded content + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudfront Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this codec. Examples include "UTF-8" and +"CP1252" + +JSON requires valid UTF-8 strings, but in some cases, software that +emits JSON does so in another encoding (nxlog, for example). In +weird cases like this, you can set the charset setting to the +actual encoding of the text and logstash will convert it for you. + +For nxlog users, you'll want to set this to "CP1252" + + diff --git a/docs/versioned-plugins/codecs/cloudfront-v7.2.0.asciidoc b/docs/versioned-plugins/codecs/cloudfront-v7.2.0.asciidoc new file mode 100644 index 000000000..2b441d683 --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudfront-v7.2.0.asciidoc @@ -0,0 +1,53 @@ +:integration: aws +:plugin: cloudfront +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudfront codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This codec will read cloudfront encoded content + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudfront Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this codec. Examples include "UTF-8" and +"CP1252" + +JSON requires valid UTF-8 strings, but in some cases, software that +emits JSON does so in another encoding (nxlog, for example). In +weird cases like this, you can set the charset setting to the +actual encoding of the text and logstash will convert it for you. + +For nxlog users, you'll want to set this to "CP1252" + + diff --git a/docs/versioned-plugins/codecs/cloudfront-v7.2.1.asciidoc b/docs/versioned-plugins/codecs/cloudfront-v7.2.1.asciidoc new file mode 100644 index 000000000..64d2be121 --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudfront-v7.2.1.asciidoc @@ -0,0 +1,53 @@ +:integration: aws +:plugin: cloudfront +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudfront codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This codec will read cloudfront encoded content + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudfront Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this codec. Examples include "UTF-8" and +"CP1252" + +JSON requires valid UTF-8 strings, but in some cases, software that +emits JSON does so in another encoding (nxlog, for example). In +weird cases like this, you can set the charset setting to the +actual encoding of the text and logstash will convert it for you. + +For nxlog users, you'll want to set this to "CP1252" + + diff --git a/docs/versioned-plugins/codecs/cloudtrail-index.asciidoc b/docs/versioned-plugins/codecs/cloudtrail-index.asciidoc index 065665807..8463ecc3a 100644 --- a/docs/versioned-plugins/codecs/cloudtrail-index.asciidoc +++ b/docs/versioned-plugins/codecs/cloudtrail-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -20,6 +24,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::cloudtrail-v7.2.1.asciidoc[] +include::cloudtrail-v7.2.0.asciidoc[] +include::cloudtrail-v7.1.8.asciidoc[] +include::cloudtrail-v7.1.7.asciidoc[] include::cloudtrail-v7.1.6.asciidoc[] include::cloudtrail-v7.1.5.asciidoc[] include::cloudtrail-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/codecs/cloudtrail-v7.1.7.asciidoc b/docs/versioned-plugins/codecs/cloudtrail-v7.1.7.asciidoc new file mode 100644 index 000000000..dd484961f --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudtrail-v7.1.7.asciidoc @@ -0,0 +1,45 @@ +:integration: aws +:plugin: cloudtrail +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudtrail codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This is the base class for logstash codecs. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudtrail Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + + + + diff --git a/docs/versioned-plugins/codecs/cloudtrail-v7.1.8.asciidoc b/docs/versioned-plugins/codecs/cloudtrail-v7.1.8.asciidoc new file mode 100644 index 000000000..05eb818c9 --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudtrail-v7.1.8.asciidoc @@ -0,0 +1,45 @@ +:integration: aws +:plugin: cloudtrail +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudtrail codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This is the base class for logstash codecs. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudtrail Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + + + + diff --git a/docs/versioned-plugins/codecs/cloudtrail-v7.2.0.asciidoc b/docs/versioned-plugins/codecs/cloudtrail-v7.2.0.asciidoc new file mode 100644 index 000000000..9314b9edc --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudtrail-v7.2.0.asciidoc @@ -0,0 +1,45 @@ +:integration: aws +:plugin: cloudtrail +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudtrail codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This is the base class for logstash codecs. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudtrail Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + + + + diff --git a/docs/versioned-plugins/codecs/cloudtrail-v7.2.1.asciidoc b/docs/versioned-plugins/codecs/cloudtrail-v7.2.1.asciidoc new file mode 100644 index 000000000..46f62806c --- /dev/null +++ b/docs/versioned-plugins/codecs/cloudtrail-v7.2.1.asciidoc @@ -0,0 +1,45 @@ +:integration: aws +:plugin: cloudtrail +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudtrail codec plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This is the base class for logstash codecs. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudtrail Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + + + + diff --git a/docs/versioned-plugins/codecs/fluent-index.asciidoc b/docs/versioned-plugins/codecs/fluent-index.asciidoc index e27ab8e6c..911ec6693 100644 --- a/docs/versioned-plugins/codecs/fluent-index.asciidoc +++ b/docs/versioned-plugins/codecs/fluent-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-06-25 | <> | 2023-04-04 | <> | 2021-09-22 | <> | 2021-08-09 @@ -16,6 +17,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::fluent-v3.4.3.asciidoc[] include::fluent-v3.4.2.asciidoc[] include::fluent-v3.4.1.asciidoc[] include::fluent-v3.4.0.asciidoc[] diff --git a/docs/versioned-plugins/codecs/fluent-v3.4.3.asciidoc b/docs/versioned-plugins/codecs/fluent-v3.4.3.asciidoc new file mode 100644 index 000000000..9221fa6a5 --- /dev/null +++ b/docs/versioned-plugins/codecs/fluent-v3.4.3.asciidoc @@ -0,0 +1,83 @@ +:plugin: fluent +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.4.3 +:release_date: 2024-06-25 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-fluent/blob/v3.4.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Fluent codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This codec handles fluentd's msgpack schema. + +For example, you can receive logs from `fluent-logger-ruby` with: +[source,ruby] + input { + tcp { + codec => fluent + port => 4000 + } + } + +And from your ruby code in your own application: +[source,ruby] + logger = Fluent::Logger::FluentLogger.new(nil, :host => "example.log", :port => 4000) + logger.post("some_tag", { "your" => "data", "here" => "yay!" }) + + +NOTE: Fluent uses second-precision for events, so you will not see sub-second precision +on events processed by this codec. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Fluent Codec configuration options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-nanosecond_precision>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-nanosecond_precision"] +===== `nanosecond_precision` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables sub-second level precision while encoding events. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the decoded values. If this setting is not +set, data will be stored at the root (top level) of the event. + +For example, if you want data to be put under the `logs` field: +[source,ruby] + input { + tcp { + codec => fluent { + target => "[logs]" + } + port => 4000 + } + } + diff --git a/docs/versioned-plugins/codecs/json_lines-index.asciidoc b/docs/versioned-plugins/codecs/json_lines-index.asciidoc index fd54b3990..569d4cdad 100644 --- a/docs/versioned-plugins/codecs/json_lines-index.asciidoc +++ b/docs/versioned-plugins/codecs/json_lines-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-09-06 +| <> | 2024-09-05 +| <> | 2024-08-29 | <> | 2021-07-01 | <> | 2018-03-22 | <> | 2017-11-07 @@ -12,6 +15,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::json_lines-v3.2.2.asciidoc[] +include::json_lines-v3.2.1.asciidoc[] +include::json_lines-v3.2.0.asciidoc[] include::json_lines-v3.1.0.asciidoc[] include::json_lines-v3.0.6.asciidoc[] include::json_lines-v3.0.5.asciidoc[] diff --git a/docs/versioned-plugins/codecs/json_lines-v3.2.0.asciidoc b/docs/versioned-plugins/codecs/json_lines-v3.2.0.asciidoc new file mode 100644 index 000000000..0ac7bb10e --- /dev/null +++ b/docs/versioned-plugins/codecs/json_lines-v3.2.0.asciidoc @@ -0,0 +1,100 @@ +:plugin: json_lines +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.2.0 +:release_date: 2024-08-29 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-json_lines/blob/v3.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Json_lines codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This codec will decode streamed JSON that is newline delimited. +Encoding will emit a single JSON string ending in a `@delimiter` +NOTE: Do not use this codec if your source input is line-oriented JSON, for +example, redis or file inputs. Rather, use the json codec. +More info: This codec is expecting to receive a stream (string) of newline +terminated lines. The file input will produce a line string without a newline. +Therefore this codec cannot work with line oriented inputs. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Json_lines Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +| <<{version}-plugins-{type}s-{plugin}-delimiter>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this codec. Examples include `UTF-8` and +`CP1252` + +JSON requires valid `UTF-8` strings, but in some cases, software that +emits JSON does so in another encoding (nxlog, for example). In +weird cases like this, you can set the charset setting to the +actual encoding of the text and logstash will convert it for you. + +For nxlog users, you'll want to set this to `CP1252` + +[id="{version}-plugins-{type}s-{plugin}-delimiter"] +===== `delimiter` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\n"` + +Change the delimiter that separates lines + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names + ** `v1`, `v8`: Elastic Common Schema compliant behavior (warns when `target` isn't set) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the parsed data. If this setting is not +set, the JSON data will be stored at the root (top level) of the event. + +For example, if you want data to be put under the `document` field: +[source,ruby] + input { + http { + codec => json_lines { + target => "[document]" + } + } + } + diff --git a/docs/versioned-plugins/codecs/json_lines-v3.2.1.asciidoc b/docs/versioned-plugins/codecs/json_lines-v3.2.1.asciidoc new file mode 100644 index 000000000..16ed87820 --- /dev/null +++ b/docs/versioned-plugins/codecs/json_lines-v3.2.1.asciidoc @@ -0,0 +1,109 @@ +:plugin: json_lines +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.2.1 +:release_date: 2024-09-05 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-json_lines/blob/v3.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Json_lines codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This codec will decode streamed JSON that is newline delimited. +Encoding will emit a single JSON string ending in a `@delimiter` +NOTE: Do not use this codec if your source input is line-oriented JSON, for +example, redis or file inputs. Rather, use the json codec. +More info: This codec is expecting to receive a stream (string) of newline +terminated lines. The file input will produce a line string without a newline. +Therefore this codec cannot work with line oriented inputs. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Json_lines Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +| <<{version}-plugins-{type}s-{plugin}-decode_size_limit_bytes>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-delimiter>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this codec. Examples include `UTF-8` and +`CP1252` + +JSON requires valid `UTF-8` strings, but in some cases, software that +emits JSON does so in another encoding (nxlog, for example). In +weird cases like this, you can set the charset setting to the +actual encoding of the text and logstash will convert it for you. + +For nxlog users, you'll want to set this to `CP1252` + +[id="{version}-plugins-{type}s-{plugin}-decode_size_limit_bytes"] +===== `decode_size_limit_bytes` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is 512 MB + +Maximum number of bytes for a single line before stop processing. + +[id="{version}-plugins-{type}s-{plugin}-delimiter"] +===== `delimiter` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\n"` + +Change the delimiter that separates lines + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names + ** `v1`, `v8`: Elastic Common Schema compliant behavior (warns when `target` isn't set) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the parsed data. If this setting is not +set, the JSON data will be stored at the root (top level) of the event. + +For example, if you want data to be put under the `document` field: +[source,ruby] + input { + http { + codec => json_lines { + target => "[document]" + } + } + } + diff --git a/docs/versioned-plugins/codecs/json_lines-v3.2.2.asciidoc b/docs/versioned-plugins/codecs/json_lines-v3.2.2.asciidoc new file mode 100644 index 000000000..439001115 --- /dev/null +++ b/docs/versioned-plugins/codecs/json_lines-v3.2.2.asciidoc @@ -0,0 +1,109 @@ +:plugin: json_lines +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.2.2 +:release_date: 2024-09-06 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-json_lines/blob/v3.2.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Json_lines codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This codec will decode streamed JSON that is newline delimited. +Encoding will emit a single JSON string ending in a `@delimiter` +NOTE: Do not use this codec if your source input is line-oriented JSON, for +example, redis or file inputs. Rather, use the json codec. +More info: This codec is expecting to receive a stream (string) of newline +terminated lines. The file input will produce a line string without a newline. +Therefore this codec cannot work with line oriented inputs. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Json_lines Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +| <<{version}-plugins-{type}s-{plugin}-decode_size_limit_bytes>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-delimiter>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this codec. Examples include `UTF-8` and +`CP1252` + +JSON requires valid `UTF-8` strings, but in some cases, software that +emits JSON does so in another encoding (nxlog, for example). In +weird cases like this, you can set the charset setting to the +actual encoding of the text and logstash will convert it for you. + +For nxlog users, you'll want to set this to `CP1252` + +[id="{version}-plugins-{type}s-{plugin}-decode_size_limit_bytes"] +===== `decode_size_limit_bytes` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is 512 MB + +Maximum number of bytes for a single line before stop processing. + +[id="{version}-plugins-{type}s-{plugin}-delimiter"] +===== `delimiter` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\n"` + +Change the delimiter that separates lines + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names + ** `v1`, `v8`: Elastic Common Schema compliant behavior (warns when `target` isn't set) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the parsed data. If this setting is not +set, the JSON data will be stored at the root (top level) of the event. + +For example, if you want data to be put under the `document` field: +[source,ruby] + input { + http { + codec => json_lines { + target => "[document]" + } + } + } + diff --git a/docs/versioned-plugins/codecs/multiline-index.asciidoc b/docs/versioned-plugins/codecs/multiline-index.asciidoc index e0ed08c25..f6898940b 100644 --- a/docs/versioned-plugins/codecs/multiline-index.asciidoc +++ b/docs/versioned-plugins/codecs/multiline-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-04-25 | <> | 2021-09-28 | <> | 2021-07-27 | <> | 2021-06-23 @@ -16,6 +17,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::multiline-v3.1.2.asciidoc[] include::multiline-v3.1.1.asciidoc[] include::multiline-v3.1.0.asciidoc[] include::multiline-v3.0.11.asciidoc[] diff --git a/docs/versioned-plugins/codecs/multiline-v3.1.2.asciidoc b/docs/versioned-plugins/codecs/multiline-v3.1.2.asciidoc new file mode 100644 index 000000000..078391843 --- /dev/null +++ b/docs/versioned-plugins/codecs/multiline-v3.1.2.asciidoc @@ -0,0 +1,236 @@ +:plugin: multiline +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.1.2 +:release_date: 2024-04-25 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-multiline/blob/v3.1.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Multiline codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The multiline codec will collapse multiline messages and merge them into a +single event. + +IMPORTANT: If you are using a Logstash input plugin that supports multiple +hosts, such as the {logstash-ref}/plugins-inputs-beats.html[beats input plugin], you should not use +the multiline codec to handle multiline events. Doing so may result in the +mixing of streams and corrupted event data. In this situation, you need to +handle multiline events before sending the event data to Logstash. + +The original goal of this codec was to allow joining of multiline messages +from files into a single event. For example, joining Java exception and +stacktrace messages into a single event. + +The config looks like this: +[source,ruby] + input { + stdin { + codec => multiline { + pattern => "pattern, a regexp" + negate => "true" or "false" + what => "previous" or "next" + } + } + } + +The `pattern` should match what you believe to be an indicator that the field +is part of a multi-line event. + +The `what` must be `previous` or `next` and indicates the relation +to the multi-line event. + +The `negate` can be `true` or `false` (defaults to `false`). If `true`, a +message not matching the pattern will constitute a match of the multiline +filter and the `what` will be applied. (vice-versa is also true) + +For example, Java stack traces are multiline and usually have the message +starting at the far-left, with each subsequent line indented. Do this: +[source,ruby] + input { + stdin { + codec => multiline { + pattern => "^\s" + what => "previous" + } + } + } + +This says that any line starting with whitespace belongs to the previous line. + +Another example is to merge lines not starting with a date up to the previous +line.. +[source,ruby] + input { + file { + path => "/var/log/someapp.log" + codec => multiline { + # Grok pattern names are valid! :) + pattern => "^%{TIMESTAMP_ISO8601} " + negate => true + what => "previous" + } + } + } + +This says that any line not starting with a timestamp should be merged with the previous line. + +One more common example is C line continuations (backslash). Here's how to do that: +[source,ruby] + input { + stdin { + codec => multiline { + pattern => "\\$" + what => "next" + } + } + } + +This says that any line ending with a backslash should be combined with the +following line. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Multiline codec configuration options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_flush_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-charset>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_bytes>> |{logstash-ref}/configuration-file-structure.html#bytes[bytes]|No +| <<{version}-plugins-{type}s-{plugin}-max_lines>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-multiline_tag>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-negate>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-patterns_dir>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-what>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["previous", "next"]`|Yes +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_flush_interval"] +===== `auto_flush_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The accumulation of multiple lines will be converted to an event when either a +matching new line is seen or there has been no new data appended for this many +seconds. No default. If unset, no auto_flush. Units: seconds + +[id="{version}-plugins-{type}s-{plugin}-charset"] +===== `charset` + + * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale` + * Default value is `"UTF-8"` + +The character encoding used in this input. Examples include `UTF-8` +and `cp1252` + +This setting is useful if your log files are in `Latin-1` (aka `cp1252`) +or in another character set other than `UTF-8`. + +This only affects "plain" format logs since JSON is `UTF-8` already. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: plugin only sets the `message` field + ** `v1`,`v8`: Elastic Common Schema compliant behavior (`[event][original]` is also added) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-max_bytes"] +===== `max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#bytes[bytes] + * Default value is `"10 MiB"` + +The accumulation of events can make logstash exit with an out of memory error +if event boundaries are not correctly defined. This settings make sure to flush +multiline events after reaching a number of bytes, it is used in combination +max_lines. + +[id="{version}-plugins-{type}s-{plugin}-max_lines"] +===== `max_lines` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The accumulation of events can make logstash exit with an out of memory error +if event boundaries are not correctly defined. This settings make sure to flush +multiline events after reaching a number of lines, it is used in combination +max_bytes. + +[id="{version}-plugins-{type}s-{plugin}-multiline_tag"] +===== `multiline_tag` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"multiline"` + +Tag multiline events with a given tag. This tag will only be added +to events that actually have multiple lines in them. + +[id="{version}-plugins-{type}s-{plugin}-negate"] +===== `negate` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Negate the regexp pattern ('if not matched'). + +[id="{version}-plugins-{type}s-{plugin}-pattern"] +===== `pattern` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The regular expression to match. + +[id="{version}-plugins-{type}s-{plugin}-patterns_dir"] +===== `patterns_dir` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Logstash ships by default with a bunch of patterns, so you don't +necessarily need to define this yourself unless you are adding additional +patterns. + +Pattern files are plain text with format: +[source,ruby] + NAME PATTERN + +For example: +[source,ruby] + NUMBER \d+ + +[id="{version}-plugins-{type}s-{plugin}-what"] +===== `what` + + * This is a required setting. + * Value can be any of: `previous`, `next` + * There is no default value for this setting. + +If the pattern matched, does event belong to the next or previous event? + + diff --git a/docs/versioned-plugins/codecs/netflow-index.asciidoc b/docs/versioned-plugins/codecs/netflow-index.asciidoc index 60863a875..d9e1b4b9d 100644 --- a/docs/versioned-plugins/codecs/netflow-index.asciidoc +++ b/docs/versioned-plugins/codecs/netflow-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2023-12-22 +| <> | 2023-12-14 +| <> | 2022-09-26 | <> | 2021-08-01 | <> | 2018-12-30 | <> | 2018-10-28 @@ -40,6 +43,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::netflow-v4.3.2.asciidoc[] +include::netflow-v4.3.1.asciidoc[] +include::netflow-v4.3.0.asciidoc[] include::netflow-v4.2.2.asciidoc[] include::netflow-v4.2.1.asciidoc[] include::netflow-v4.2.0.asciidoc[] diff --git a/docs/versioned-plugins/codecs/netflow-v4.3.0.asciidoc b/docs/versioned-plugins/codecs/netflow-v4.3.0.asciidoc new file mode 100644 index 000000000..c379db853 --- /dev/null +++ b/docs/versioned-plugins/codecs/netflow-v4.3.0.asciidoc @@ -0,0 +1,218 @@ +:plugin: netflow +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.3.0 +:release_date: 2022-09-26 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-netflow/blob/v4.3.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Netflow codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows. + +==== Supported Netflow/IPFIX exporters + +This codec supports: + +* Netflow v5 +* Netflow v9 +* IPFIX + +The following Netflow/IPFIX exporters have been seen and tested with the most recent version of the Netflow Codec: + +[cols="6,^2,^2,^2,12",options="header"] +|=========================================================================================== +|Netflow exporter | v5 | v9 | IPFIX | Remarks +|Barracuda Firewall | | | y | With support for Extended Uniflow +|Cisco ACI | | y | | +|Cisco ASA | | y | | +|Cisco ASR 1k | | | N | Fails because of duplicate fields +|Cisco ASR 9k | | y | | +|Cisco IOS 12.x | | y | | +|Cisco ISR w/ HSL | | N | | Fails because of duplicate fields, see: https://github.com/logstash-plugins/logstash-codec-netflow/issues/93 +|Cisco WLC | | y | | +|Citrix Netscaler | | | y | Still some unknown fields, labeled netscalerUnknown +|fprobe | y | | | +|Fortigate FortiOS | | y | | +|Huawei Netstream | | y | | +|ipt_NETFLOW | y | y | y | +|IXIA packet broker | | | y | +|Juniper MX | y | | y | SW > 12.3R8. Fails to decode IPFIX from Junos 16.1 due to duplicate field names which we currently don't support. +|Mikrotik | y | | y | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow +|nProbe | y | y | y | L7 DPI fields now also supported +|Nokia BRAS | | | y | +|OpenBSD pflow | y | N | y | http://man.openbsd.org/OpenBSD-current/man4/pflow.4 +|Riverbed | | N | | Not supported due to field ID conflicts. Workaround available in the definitions directory over at Elastiflow +|Sandvine Procera PacketLogic| | | y | v15.1 +|Softflowd | y | y | y | IPFIX supported in https://github.com/djmdjm/softflowd +|Sophos UTM | | | y | +|Streamcore Streamgroomer | | y | | +|Palo Alto PAN-OS | | y | | +|Ubiquiti Edgerouter X | | y | | With MPLS labels +|VMware VDS | | | y | Still some unknown fields +|YAF | | | y | With silk and applabel, but no DPI plugin support +|vIPtela | | | y | +|=========================================================================================== + +==== Usage + +Example Logstash configuration that will listen on 2055/udp for Netflow v5,v9 and IPFIX: + +[source, ruby] +-------------------------- +input { + udp { + port => 2055 + codec => netflow + } +} +-------------------------- + +For high-performance production environments the configuration below will decode up to 15000 flows/sec from a Cisco ASR 9000 router on a dedicated 16 CPU instance. If your total flowrate exceeds 15000 flows/sec, you should use multiple Logstash instances. + +Note that for richer flows from a Cisco ASA firewall this number will be at least 3x lower. + +[source, ruby] +-------------------------- +input { + udp { + port => 2055 + codec => netflow + receive_buffer_bytes => 16777216 + workers => 16 + } +-------------------------- + +To mitigate dropped packets, make sure to increase the Linux kernel receive buffer limit: + + # sysctl -w net.core.rmem_max=$((1024*1024*16)) + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Netflow Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_save_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-cache_ttl>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-include_flowset_id>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ipfix_definitions>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-netflow_definitions>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-versions>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_save_path"] +===== `cache_save_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Enables the template cache and saves it in the specified directory. This +minimizes data loss after Logstash restarts because the codec doesn't have to +wait for the arrival of templates, but instead reload already received +templates received during previous runs. + +Template caches are saved as: + +* {logstash-ref}/configuration-file-structure.html#path[path]/netflow_templates.cache for Netflow v9 templates. +* {logstash-ref}/configuration-file-structure.html#path[path]/ipfix_templates.cache for IPFIX templates. + +[id="{version}-plugins-{type}s-{plugin}-cache_ttl"] +===== `cache_ttl` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4000` + +Netflow v9/v10 template cache TTL (seconds) + +[id="{version}-plugins-{type}s-{plugin}-include_flowset_id"] +===== `include_flowset_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Only makes sense for ipfix, v9 already includes this +Setting to true will include the flowset_id in events +Allows you to work with sequences, for instance with the aggregate filter + +[id="{version}-plugins-{type}s-{plugin}-ipfix_definitions"] +===== `ipfix_definitions` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Override YAML file containing IPFIX field definitions + +Very similar to the Netflow version except there is a top level Private +Enterprise Number (PEN) key added: + +[source,yaml] +-------------------------- +pen: +id: +- :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string +- :name +id: +- :skip +-------------------------- + +There is an implicit PEN 0 for the standard fields. + +See for the base set. + +[id="{version}-plugins-{type}s-{plugin}-netflow_definitions"] +===== `netflow_definitions` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Override YAML file containing Netflow field definitions + +Each Netflow field is defined like so: + +[source,yaml] +-------------------------- +id: +- default length in bytes +- :name +id: +- :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string +- :name +id: +- :skip +-------------------------- + +See for the base set. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"netflow"` + +Specify into what field you want the Netflow data. + +[id="{version}-plugins-{type}s-{plugin}-versions"] +===== `versions` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[5, 9, 10]` + +Specify which Netflow versions you will accept. + + diff --git a/docs/versioned-plugins/codecs/netflow-v4.3.1.asciidoc b/docs/versioned-plugins/codecs/netflow-v4.3.1.asciidoc new file mode 100644 index 000000000..dbfa55faf --- /dev/null +++ b/docs/versioned-plugins/codecs/netflow-v4.3.1.asciidoc @@ -0,0 +1,218 @@ +:plugin: netflow +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.3.1 +:release_date: 2023-12-14 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-netflow/blob/v4.3.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Netflow codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows. + +==== Supported Netflow/IPFIX exporters + +This codec supports: + +* Netflow v5 +* Netflow v9 +* IPFIX + +The following Netflow/IPFIX exporters have been seen and tested with the most recent version of the Netflow Codec: + +[cols="6,^2,^2,^2,12",options="header"] +|=========================================================================================== +|Netflow exporter | v5 | v9 | IPFIX | Remarks +|Barracuda Firewall | | | y | With support for Extended Uniflow +|Cisco ACI | | y | | +|Cisco ASA | | y | | +|Cisco ASR 1k | | | N | Fails because of duplicate fields +|Cisco ASR 9k | | y | | +|Cisco IOS 12.x | | y | | +|Cisco ISR w/ HSL | | N | | Fails because of duplicate fields, see: https://github.com/logstash-plugins/logstash-codec-netflow/issues/93 +|Cisco WLC | | y | | +|Citrix Netscaler | | | y | Still some unknown fields, labeled netscalerUnknown +|fprobe | y | | | +|Fortigate FortiOS | | y | | +|Huawei Netstream | | y | | +|ipt_NETFLOW | y | y | y | +|IXIA packet broker | | | y | +|Juniper MX | y | | y | SW > 12.3R8. Fails to decode IPFIX from Junos 16.1 due to duplicate field names which we currently don't support. +|Mikrotik | y | | y | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow +|nProbe | y | y | y | L7 DPI fields now also supported +|Nokia BRAS | | | y | +|OpenBSD pflow | y | N | y | http://man.openbsd.org/OpenBSD-current/man4/pflow.4 +|Riverbed | | N | | Not supported due to field ID conflicts. Workaround available in the definitions directory over at Elastiflow +|Sandvine Procera PacketLogic| | | y | v15.1 +|Softflowd | y | y | y | IPFIX supported in https://github.com/djmdjm/softflowd +|Sophos UTM | | | y | +|Streamcore Streamgroomer | | y | | +|Palo Alto PAN-OS | | y | | +|Ubiquiti Edgerouter X | | y | | With MPLS labels +|VMware VDS | | | y | Still some unknown fields +|YAF | | | y | With silk and applabel, but no DPI plugin support +|vIPtela | | | y | +|=========================================================================================== + +==== Usage + +Example Logstash configuration that will listen on 2055/udp for Netflow v5,v9 and IPFIX: + +[source, ruby] +-------------------------- +input { + udp { + port => 2055 + codec => netflow + } +} +-------------------------- + +For high-performance production environments the configuration below will decode up to 15000 flows/sec from a Cisco ASR 9000 router on a dedicated 16 CPU instance. If your total flowrate exceeds 15000 flows/sec, you should use multiple Logstash instances. + +Note that for richer flows from a Cisco ASA firewall this number will be at least 3x lower. + +[source, ruby] +-------------------------- +input { + udp { + port => 2055 + codec => netflow + receive_buffer_bytes => 16777216 + workers => 16 + } +-------------------------- + +To mitigate dropped packets, make sure to increase the Linux kernel receive buffer limit: + + # sysctl -w net.core.rmem_max=$((1024*1024*16)) + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Netflow Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_save_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-cache_ttl>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-include_flowset_id>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ipfix_definitions>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-netflow_definitions>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-versions>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_save_path"] +===== `cache_save_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Enables the template cache and saves it in the specified directory. This +minimizes data loss after Logstash restarts because the codec doesn't have to +wait for the arrival of templates, but instead reload already received +templates received during previous runs. + +Template caches are saved as: + +* {logstash-ref}/configuration-file-structure.html#path[path]/netflow_templates.cache for Netflow v9 templates. +* {logstash-ref}/configuration-file-structure.html#path[path]/ipfix_templates.cache for IPFIX templates. + +[id="{version}-plugins-{type}s-{plugin}-cache_ttl"] +===== `cache_ttl` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4000` + +Netflow v9/v10 template cache TTL (seconds) + +[id="{version}-plugins-{type}s-{plugin}-include_flowset_id"] +===== `include_flowset_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Only makes sense for ipfix, v9 already includes this +Setting to true will include the flowset_id in events +Allows you to work with sequences, for instance with the aggregate filter + +[id="{version}-plugins-{type}s-{plugin}-ipfix_definitions"] +===== `ipfix_definitions` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Override YAML file containing IPFIX field definitions + +Very similar to the Netflow version except there is a top level Private +Enterprise Number (PEN) key added: + +[source,yaml] +-------------------------- +pen: +id: +- :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string +- :name +id: +- :skip +-------------------------- + +There is an implicit PEN 0 for the standard fields. + +See for the base set. + +[id="{version}-plugins-{type}s-{plugin}-netflow_definitions"] +===== `netflow_definitions` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Override YAML file containing Netflow field definitions + +Each Netflow field is defined like so: + +[source,yaml] +-------------------------- +id: +- default length in bytes +- :name +id: +- :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string +- :name +id: +- :skip +-------------------------- + +See for the base set. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"netflow"` + +Specify into what field you want the Netflow data. + +[id="{version}-plugins-{type}s-{plugin}-versions"] +===== `versions` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[5, 9, 10]` + +Specify which Netflow versions you will accept. + + diff --git a/docs/versioned-plugins/codecs/netflow-v4.3.2.asciidoc b/docs/versioned-plugins/codecs/netflow-v4.3.2.asciidoc new file mode 100644 index 000000000..bceb32da6 --- /dev/null +++ b/docs/versioned-plugins/codecs/netflow-v4.3.2.asciidoc @@ -0,0 +1,218 @@ +:plugin: netflow +:type: codec + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.3.2 +:release_date: 2023-12-22 +:changelog_url: https://github.com/logstash-plugins/logstash-codec-netflow/blob/v4.3.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Netflow codec plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows. + +==== Supported Netflow/IPFIX exporters + +This codec supports: + +* Netflow v5 +* Netflow v9 +* IPFIX + +The following Netflow/IPFIX exporters have been seen and tested with the most recent version of the Netflow Codec: + +[cols="6,^2,^2,^2,12",options="header"] +|=========================================================================================== +|Netflow exporter | v5 | v9 | IPFIX | Remarks +|Barracuda Firewall | | | y | With support for Extended Uniflow +|Cisco ACI | | y | | +|Cisco ASA | | y | | +|Cisco ASR 1k | | | N | Fails because of duplicate fields +|Cisco ASR 9k | | y | | +|Cisco IOS 12.x | | y | | +|Cisco ISR w/ HSL | | N | | Fails because of duplicate fields, see: https://github.com/logstash-plugins/logstash-codec-netflow/issues/93 +|Cisco WLC | | y | | +|Citrix Netscaler | | | y | Still some unknown fields, labeled netscalerUnknown +|fprobe | y | | | +|Fortigate FortiOS | | y | | +|Huawei Netstream | | y | | +|ipt_NETFLOW | y | y | y | +|IXIA packet broker | | | y | +|Juniper MX | y | | y | SW > 12.3R8. Fails to decode IPFIX from Junos 16.1 due to duplicate field names which we currently don't support. +|Mikrotik | y | | y | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow +|nProbe | y | y | y | L7 DPI fields now also supported +|Nokia BRAS | | | y | +|OpenBSD pflow | y | N | y | http://man.openbsd.org/OpenBSD-current/man4/pflow.4 +|Riverbed | | N | | Not supported due to field ID conflicts. Workaround available in the definitions directory over at Elastiflow +|Sandvine Procera PacketLogic| | | y | v15.1 +|Softflowd | y | y | y | IPFIX supported in https://github.com/djmdjm/softflowd +|Sophos UTM | | | y | +|Streamcore Streamgroomer | | y | | +|Palo Alto PAN-OS | | y | | +|Ubiquiti Edgerouter X | | y | | With MPLS labels +|VMware VDS | | | y | Still some unknown fields +|YAF | | | y | With silk and applabel, but no DPI plugin support +|vIPtela | | | y | +|=========================================================================================== + +==== Usage + +Example Logstash configuration that will listen on 2055/udp for Netflow v5,v9 and IPFIX: + +[source, ruby] +-------------------------- +input { + udp { + port => 2055 + codec => netflow + } +} +-------------------------- + +For high-performance production environments the configuration below will decode up to 15000 flows/sec from a Cisco ASR 9000 router on a dedicated 16 CPU instance. If your total flowrate exceeds 15000 flows/sec, you should use multiple Logstash instances. + +Note that for richer flows from a Cisco ASA firewall this number will be at least 3x lower. + +[source, ruby] +-------------------------- +input { + udp { + port => 2055 + codec => netflow + receive_buffer_bytes => 16777216 + workers => 16 + } +-------------------------- + +To mitigate dropped packets, make sure to increase the Linux kernel receive buffer limit: + + # sysctl -w net.core.rmem_max=$((1024*1024*16)) + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Netflow Codec Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_save_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-cache_ttl>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-include_flowset_id>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ipfix_definitions>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-netflow_definitions>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-versions>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_save_path"] +===== `cache_save_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Enables the template cache and saves it in the specified directory. This +minimizes data loss after Logstash restarts because the codec doesn't have to +wait for the arrival of templates, but instead reload already received +templates received during previous runs. + +Template caches are saved as: + +* {logstash-ref}/configuration-file-structure.html#path[path]/netflow_templates.cache for Netflow v9 templates. +* {logstash-ref}/configuration-file-structure.html#path[path]/ipfix_templates.cache for IPFIX templates. + +[id="{version}-plugins-{type}s-{plugin}-cache_ttl"] +===== `cache_ttl` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4000` + +Netflow v9/v10 template cache TTL (seconds) + +[id="{version}-plugins-{type}s-{plugin}-include_flowset_id"] +===== `include_flowset_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Only makes sense for ipfix, v9 already includes this +Setting to true will include the flowset_id in events +Allows you to work with sequences, for instance with the aggregate filter + +[id="{version}-plugins-{type}s-{plugin}-ipfix_definitions"] +===== `ipfix_definitions` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Override YAML file containing IPFIX field definitions + +Very similar to the Netflow version except there is a top level Private +Enterprise Number (PEN) key added: + +[source,yaml] +-------------------------- +pen: +id: +- :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string +- :name +id: +- :skip +-------------------------- + +There is an implicit PEN 0 for the standard fields. + +See for the base set. + +[id="{version}-plugins-{type}s-{plugin}-netflow_definitions"] +===== `netflow_definitions` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Override YAML file containing Netflow field definitions + +Each Netflow field is defined like so: + +[source,yaml] +-------------------------- +id: +- default length in bytes +- :name +id: +- :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string +- :name +id: +- :skip +-------------------------- + +See for the base set. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"netflow"` + +Specify into what field you want the Netflow data. + +[id="{version}-plugins-{type}s-{plugin}-versions"] +===== `versions` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[5, 9, 10]` + +Specify which Netflow versions you will accept. + + diff --git a/docs/versioned-plugins/filters/de_dot-index.asciidoc b/docs/versioned-plugins/filters/de_dot-index.asciidoc index 6aa1261e5..3747d571e 100644 --- a/docs/versioned-plugins/filters/de_dot-index.asciidoc +++ b/docs/versioned-plugins/filters/de_dot-index.asciidoc @@ -5,12 +5,14 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-05-27 | <> | 2019-02-04 | <> | 2017-11-07 | <> | 2017-08-15 | <> | 2017-06-23 |======================================================================= +include::de_dot-v1.1.0.asciidoc[] include::de_dot-v1.0.4.asciidoc[] include::de_dot-v1.0.3.asciidoc[] include::de_dot-v1.0.2.asciidoc[] diff --git a/docs/versioned-plugins/filters/de_dot-v1.1.0.asciidoc b/docs/versioned-plugins/filters/de_dot-v1.1.0.asciidoc new file mode 100644 index 000000000..e0a4d6622 --- /dev/null +++ b/docs/versioned-plugins/filters/de_dot-v1.1.0.asciidoc @@ -0,0 +1,91 @@ +:plugin: de_dot +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.1.0 +:release_date: 2024-05-27 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-de_dot/blob/v1.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== De_dot filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This filter _appears_ to rename fields by replacing `.` characters with a different +separator. In reality, it's a somewhat expensive filter that has to copy the +source field contents to a new destination field (whose name no longer contains +dots), and then remove the corresponding source field. + +It should only be used if no other options are available. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== De_dot Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-nested>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-recursive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-separator>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +The `fields` array should contain a list of known fields to act on. +If undefined, all top-level fields will be checked. Sub-fields must be +manually specified in the array. For example: `["field.suffix","[foo][bar.suffix]"]` +will result in "field_suffix" and nested or sub field ["foo"]["bar_suffix"] + +WARNING: This is an expensive operation. + +[id="{version}-plugins-{type}s-{plugin}-nested"] +===== `nested` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `nested` is _true_, then create sub-fields instead of replacing dots with +a different separator. + +[id="{version}-plugins-{type}s-{plugin}-recursive"] +===== `recursive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `recursive` is _true_, then recursively check sub-fields. It is recommended you +only use this when setting specific fields, as this is an expensive operation. + +[id="{version}-plugins-{type}s-{plugin}-separator"] +===== `separator` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"_"` + +Replace dots with this value. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] \ No newline at end of file diff --git a/docs/versioned-plugins/filters/elastic_integration-index.asciidoc b/docs/versioned-plugins/filters/elastic_integration-index.asciidoc index 12c288364..457c6d386 100644 --- a/docs/versioned-plugins/filters/elastic_integration-index.asciidoc +++ b/docs/versioned-plugins/filters/elastic_integration-index.asciidoc @@ -5,11 +5,77 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-07-16 +| <> | 2025-07-10 +| <> | 2025-07-16 +| <> | 2025-06-28 +| <> | 2025-04-28 +| <> | 2025-07-16 +| <> | 2025-07-10 +| <> | 2025-07-16 +| <> | 2025-06-28 +| <> | 2025-04-17 +| <> | 2025-07-16 +| <> | 2025-06-28 +| <> | 2025-01-28 +| <> | 2025-01-08 +| <> | 2025-01-28 +| <> | 2025-01-08 +| <> | 2024-12-07 +| <> | 2024-10-30 +| <> | 2024-10-11 +| <> | 2024-09-23 +| <> | 2024-07-16 +| <> | 2024-07-09 +| <> | 2024-07-02 +| <> | 2024-06-18 +| <> | 2024-05-14 +| <> | 2024-03-07 +| <> | 2024-03-05 +| <> | 2024-02-08 +| <> | 2024-01-31 +| <> | 2024-01-25 +| <> | 2023-12-20 +| <> | 2023-12-12 +| <> | 2023-12-04 | <> | 2023-10-04 | <> | 2023-05-10 | <> | 2023-04-14 |======================================================================= +include::elastic_integration-v9.1.1.asciidoc[] +include::elastic_integration-v9.1.0.asciidoc[] +include::elastic_integration-v9.0.2.asciidoc[] +include::elastic_integration-v9.0.1.asciidoc[] +include::elastic_integration-v9.0.0.asciidoc[] +include::elastic_integration-v8.19.1.asciidoc[] +include::elastic_integration-v8.19.0.asciidoc[] +include::elastic_integration-v8.18.2.asciidoc[] +include::elastic_integration-v8.18.1.asciidoc[] +include::elastic_integration-v8.18.0.asciidoc[] +include::elastic_integration-v8.17.3.asciidoc[] +include::elastic_integration-v8.17.2.asciidoc[] +include::elastic_integration-v8.17.1.asciidoc[] +include::elastic_integration-v8.17.0.asciidoc[] +include::elastic_integration-v8.16.1.asciidoc[] +include::elastic_integration-v8.16.0.asciidoc[] +include::elastic_integration-v0.1.17.asciidoc[] +include::elastic_integration-v0.1.16.asciidoc[] +include::elastic_integration-v0.1.15.asciidoc[] +include::elastic_integration-v0.1.14.asciidoc[] +include::elastic_integration-v0.1.13.asciidoc[] +include::elastic_integration-v0.1.12.asciidoc[] +include::elastic_integration-v0.1.11.asciidoc[] +include::elastic_integration-v0.1.10.asciidoc[] +include::elastic_integration-v0.1.9.asciidoc[] +include::elastic_integration-v0.1.8.asciidoc[] +include::elastic_integration-v0.1.7.asciidoc[] +include::elastic_integration-v0.1.6.asciidoc[] +include::elastic_integration-v0.1.5.asciidoc[] +include::elastic_integration-v0.1.4.asciidoc[] +include::elastic_integration-v0.1.3.asciidoc[] +include::elastic_integration-v0.1.2.asciidoc[] +include::elastic_integration-v0.1.0.asciidoc[] include::elastic_integration-v0.0.3.asciidoc[] include::elastic_integration-v0.0.2.asciidoc[] include::elastic_integration-v0.0.1.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.0.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.0.asciidoc new file mode 100644 index 000000000..0575ad581 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.0.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.0 +:release_date: 2023-12-04 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Enterprise Elastic License. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `+_none+` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `+pipeline => "_none"+` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `+_ingest_pipeline_failure+`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,shell] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uriparts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `+_version+`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `+_ingest.timestamp+`, or `+_now+` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `+_@timestamp+` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `+_version+` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `+_@version+` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `+_@metadata+` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `+_tags+` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `+_id+` | `{mcc-prefix}[id]` +| `+_index+` | `{mcc-prefix}[index]` +| `+_routing+` | `{mcc-prefix}[routing]` +| `+_version+` | `{mcc-prefix}[version]` +| `+_version_type+` | `{mcc-prefix}[version_type]` +| `+_ingest.timestamp+` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled \=> false`>>. +:prohibit-ssl-verify-none: Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `ssl_verification_mode \=> none`>>. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `+ssl_enabled => false+`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `+ssl_enabled => true+`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `+ssl_enabled => false+`>> + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `+ssl_enabled => false+`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `+ssl_enabled => true+` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.10.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.10.asciidoc new file mode 100644 index 000000000..ba5ed61d7 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.10.asciidoc @@ -0,0 +1,539 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.10 +:release_date: 2024-06-18 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.10/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.11.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.11.asciidoc new file mode 100644 index 000000000..5ebbb4f77 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.11.asciidoc @@ -0,0 +1,550 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.11 +:release_date: 2024-07-02 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.11/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.12.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.12.asciidoc new file mode 100644 index 000000000..5fd334eda --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.12.asciidoc @@ -0,0 +1,550 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.12 +:release_date: 2024-07-09 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.12/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.13.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.13.asciidoc new file mode 100644 index 000000000..83cb4f610 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.13.asciidoc @@ -0,0 +1,550 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.13 +:release_date: 2024-07-16 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.13/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.14.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.14.asciidoc new file mode 100644 index 000000000..0c2b8bff5 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.14.asciidoc @@ -0,0 +1,550 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.14 +:release_date: 2024-09-23 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.14/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.15.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.15.asciidoc new file mode 100644 index 000000000..51e915230 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.15.asciidoc @@ -0,0 +1,550 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.15 +:release_date: 2024-10-11 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.15/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.16.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.16.asciidoc new file mode 100644 index 000000000..796909d2e --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.16.asciidoc @@ -0,0 +1,550 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.16 +:release_date: 2024-10-30 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.16/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.17.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.17.asciidoc new file mode 100644 index 000000000..586d56357 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.17.asciidoc @@ -0,0 +1,550 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.17 +:release_date: 2024-12-07 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.17/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.2.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.2.asciidoc new file mode 100644 index 000000000..759680fe3 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.2.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.2 +:release_date: 2023-12-12 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Enterprise Elastic License. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `+_none+` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `+pipeline => "_none"+` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `+_ingest_pipeline_failure+`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,shell] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uriparts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `+_version+`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `+_ingest.timestamp+`, or `+_now+` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `+_@timestamp+` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `+_version+` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `+_@version+` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `+_@metadata+` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `+_tags+` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `+_id+` | `{mcc-prefix}[id]` +| `+_index+` | `{mcc-prefix}[index]` +| `+_routing+` | `{mcc-prefix}[routing]` +| `+_version+` | `{mcc-prefix}[version]` +| `+_version_type+` | `{mcc-prefix}[version_type]` +| `+_ingest.timestamp+` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled \=> false`>>. +:prohibit-ssl-verify-none: Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `ssl_verification_mode \=> none`>>. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `+ssl_enabled => false+`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `+ssl_enabled => true+`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `+ssl_enabled => false+`>> + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `+ssl_enabled => false+`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `+ssl_enabled => true+` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.3.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.3.asciidoc new file mode 100644 index 000000000..6764b200a --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.3.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.3 +:release_date: 2023-12-20 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uriparts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `{mcc-prefix}[id]` +| `_index` | `{mcc-prefix}[index]` +| `_routing` | `{mcc-prefix}[routing]` +| `_version` | `{mcc-prefix}[version]` +| `_version_type` | `{mcc-prefix}[version_type]` +| `_ingest.timestamp` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.4.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.4.asciidoc new file mode 100644 index 000000000..5684fddd9 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.4.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.4 +:release_date: 2024-01-25 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uriparts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `{mcc-prefix}[id]` +| `_index` | `{mcc-prefix}[index]` +| `_routing` | `{mcc-prefix}[routing]` +| `_version` | `{mcc-prefix}[version]` +| `_version_type` | `{mcc-prefix}[version_type]` +| `_ingest.timestamp` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.5.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.5.asciidoc new file mode 100644 index 000000000..9bbd482fe --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.5.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.5 +:release_date: 2024-01-31 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uriparts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `{mcc-prefix}[id]` +| `_index` | `{mcc-prefix}[index]` +| `_routing` | `{mcc-prefix}[routing]` +| `_version` | `{mcc-prefix}[version]` +| `_version_type` | `{mcc-prefix}[version_type]` +| `_ingest.timestamp` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.6.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.6.asciidoc new file mode 100644 index 000000000..e3c5a439b --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.6.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.6 +:release_date: 2024-02-08 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uriparts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `{mcc-prefix}[id]` +| `_index` | `{mcc-prefix}[index]` +| `_routing` | `{mcc-prefix}[routing]` +| `_version` | `{mcc-prefix}[version]` +| `_version_type` | `{mcc-prefix}[version_type]` +| `_ingest.timestamp` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.7.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.7.asciidoc new file mode 100644 index 000000000..c687d62f1 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.7.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.7 +:release_date: 2024-03-05 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `{mcc-prefix}[id]` +| `_index` | `{mcc-prefix}[index]` +| `_routing` | `{mcc-prefix}[routing]` +| `_version` | `{mcc-prefix}[version]` +| `_version_type` | `{mcc-prefix}[version_type]` +| `_ingest.timestamp` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.8.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.8.asciidoc new file mode 100644 index 000000000..b66b42deb --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.8.asciidoc @@ -0,0 +1,527 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.8 +:release_date: 2024-03-07 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `{mcc-prefix}[id]` +| `_index` | `{mcc-prefix}[index]` +| `_routing` | `{mcc-prefix}[routing]` +| `_version` | `{mcc-prefix}[version]` +| `_version_type` | `{mcc-prefix}[version_type]` +| `_ingest.timestamp` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v0.1.9.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v0.1.9.asciidoc new file mode 100644 index 000000000..87f19f82a --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v0.1.9.asciidoc @@ -0,0 +1,526 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.9 +:release_date: 2024-05-14 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.16.0.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.16.0.asciidoc new file mode 100644 index 000000000..e03a17fc2 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.16.0.asciidoc @@ -0,0 +1,557 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.16.0 +:release_date: 2025-01-08 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.16.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.16.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.16.1.asciidoc new file mode 100644 index 000000000..146273503 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.16.1.asciidoc @@ -0,0 +1,557 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.16.1 +:release_date: 2025-01-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.16.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.17.0.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.17.0.asciidoc new file mode 100644 index 000000000..46c6b0805 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.17.0.asciidoc @@ -0,0 +1,557 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.17.0 +:release_date: 2025-01-08 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.17.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.17.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.17.1.asciidoc new file mode 100644 index 000000000..94a8149a6 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.17.1.asciidoc @@ -0,0 +1,557 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.17.1 +:release_date: 2025-01-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.17.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.17.2.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.17.2.asciidoc new file mode 100644 index 000000000..9ab219608 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.17.2.asciidoc @@ -0,0 +1,568 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.17.2 +:release_date: 2025-06-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.17.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.17.3.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.17.3.asciidoc new file mode 100644 index 000000000..b5731200c --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.17.3.asciidoc @@ -0,0 +1,568 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.17.3 +:release_date: 2025-07-16 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.17.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.18.0.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.18.0.asciidoc new file mode 100644 index 000000000..e01bcad48 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.18.0.asciidoc @@ -0,0 +1,623 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.18.0 +:release_date: 2025-04-17 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.18.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.18.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.18.1.asciidoc new file mode 100644 index 000000000..eadc56e57 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.18.1.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.18.1 +:release_date: 2025-06-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.18.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.18.2.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.18.2.asciidoc new file mode 100644 index 000000000..8bbff76d3 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.18.2.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.18.2 +:release_date: 2025-07-16 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.18.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.19.0.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.19.0.asciidoc new file mode 100644 index 000000000..b5ab1dfea --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.19.0.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.19.0 +:release_date: 2025-07-10 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.19.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.19.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.19.1.asciidoc new file mode 100644 index 000000000..e7aee5216 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.19.1.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.19.1 +:release_date: 2025-07-16 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.19.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.0.0.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.0.0.asciidoc new file mode 100644 index 000000000..a1d210ec1 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.0.0.asciidoc @@ -0,0 +1,623 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.0.0 +:release_date: 2025-04-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.0.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.0.1.asciidoc new file mode 100644 index 000000000..56b1990da --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.0.1.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.0.1 +:release_date: 2025-06-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.0.2.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.0.2.asciidoc new file mode 100644 index 000000000..33c05dc30 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.0.2.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.0.2 +:release_date: 2025-07-16 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.1.0.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.1.0.asciidoc new file mode 100644 index 000000000..8bb01189e --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.1.0.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.1.0 +:release_date: 2025-07-10 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.1.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.1.1.asciidoc new file mode 100644 index 000000000..931d819be --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.1.1.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.1.1 +:release_date: 2025-07-16 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.1.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-index.asciidoc b/docs/versioned-plugins/filters/elasticsearch-index.asciidoc index a9c0cfd84..30ffe6c37 100644 --- a/docs/versioned-plugins/filters/elasticsearch-index.asciidoc +++ b/docs/versioned-plugins/filters/elasticsearch-index.asciidoc @@ -5,6 +5,16 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-07-21 +| <> | 2025-05-07 +| <> | 2025-03-17 +| <> | 2025-01-23 +| <> | 2025-01-10 +| <> | 2025-07-21 +| <> | 2025-05-07 +| <> | 2025-03-17 +| <> | 2025-01-29 +| <> | 2024-12-09 | <> | 2023-09-29 | <> | 2023-09-29 | <> | 2023-09-08 @@ -38,6 +48,16 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-03 |======================================================================= +include::elasticsearch-v4.3.0.asciidoc[] +include::elasticsearch-v4.2.0.asciidoc[] +include::elasticsearch-v4.1.1.asciidoc[] +include::elasticsearch-v4.1.0.asciidoc[] +include::elasticsearch-v4.0.0.asciidoc[] +include::elasticsearch-v3.19.0.asciidoc[] +include::elasticsearch-v3.18.0.asciidoc[] +include::elasticsearch-v3.17.1.asciidoc[] +include::elasticsearch-v3.17.0.asciidoc[] +include::elasticsearch-v3.16.2.asciidoc[] include::elasticsearch-v3.16.1.asciidoc[] include::elasticsearch-v3.16.0.asciidoc[] include::elasticsearch-v3.15.3.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.16.2.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.16.2.asciidoc new file mode 100644 index 000000000..f13f36957 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v3.16.2.asciidoc @@ -0,0 +1,575 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.16.2 +:release_date: 2024-12-09 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v3.16.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-keystore>> and/or <<{version}-plugins-{type}s-{plugin}-keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of aggregation names to copy from elasticsearch response into Logstash event fields + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of docinfo fields to copy from old event (found via elasticsearch) into new event + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +An array of fields to copy from the old event (found via elasticsearch) into the +new event, currently being processed. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Filter Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +SSL + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. It can be either .jks or .p12 + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.17.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.17.0.asciidoc new file mode 100644 index 000000000..51db5ed56 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v3.17.0.asciidoc @@ -0,0 +1,586 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.17.0 +:release_date: 2025-01-29 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v3.17.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-keystore>> and/or <<{version}-plugins-{type}s-{plugin}-keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of aggregation names to copy from elasticsearch response into Logstash event fields + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of docinfo fields to copy from old event (found via elasticsearch) into new event + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +An array of fields to copy from the old event (found via elasticsearch) into the +new event, currently being processed. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Filter Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +SSL + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. It can be either .jks or .p12 + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.17.1.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.17.1.asciidoc new file mode 100644 index 000000000..4308ee363 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v3.17.1.asciidoc @@ -0,0 +1,586 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.17.1 +:release_date: 2025-03-17 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v3.17.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-keystore>> and/or <<{version}-plugins-{type}s-{plugin}-keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of aggregation names to copy from elasticsearch response into Logstash event fields + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of docinfo fields to copy from old event (found via elasticsearch) into new event + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +An array of fields to copy from the old event (found via elasticsearch) into the +new event, currently being processed. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Filter Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +SSL + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. It can be either .jks or .p12 + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.18.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.18.0.asciidoc new file mode 100644 index 000000000..f609f5fe3 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v3.18.0.asciidoc @@ -0,0 +1,632 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.18.0 +:release_date: 2025-05-07 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v3.18.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-keystore>> and/or <<{version}-plugins-{type}s-{plugin}-keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"aggregation_name" => "[path][on][event]"`: + ** `aggregation_name`: aggregation name in result from {es} + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of aggregations to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"path.in.source" => "[path][on][event]"`: + ** `path.in.source`: field path in document source of result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of docinfo (`_source`) fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + * Format: `"path.in.result" => "[path][on][event]"`: + ** `path.in.result`: field path in indexed result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of indexed fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Define the target field for placing the result data. +If this setting is omitted, the target will be the root (top level) of the event. + +The destination fields specified in <<{version}-plugins-{type}s-{plugin}-fields>>, <<{version}-plugins-{type}s-{plugin}-aggregation_fields>>, and <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> are relative to this target. + +For example, if you want the data to be put in the `operation` field: +[source,ruby] +if [type] == "end" { + filter { + query => "type:start AND transaction:%{[transactionId]}" + elasticsearch { + target => "transaction" + fields => { + "@timestamp" => "started" + "transaction_id" => "id" + } + } + } +} + +`fields` fields will be expanded into a data structure in the `target` field, overall shape looks like this: +[source,ruby] + { + "transaction" => { + "started" => "2025-04-29T12:01:46.263Z" + "id" => "1234567890" + } + } + +NOTE: when writing to a field that already exists on the event, the previous value will be overwritten. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Filter Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +SSL + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. It can be either .jks or .p12 + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.19.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.19.0.asciidoc new file mode 100644 index 000000000..217fb3509 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v3.19.0.asciidoc @@ -0,0 +1,758 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.19.0 +:release_date: 2025-07-21 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v3.19.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard {ls} field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-keystore>> and/or <<{version}-plugins-{type}s-{plugin}-keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-esql"] +==== {esql} support + +.Technical Preview +**** +The {esql} feature that allows using ES|QL queries with this plugin is in Technical Preview. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +{es} Query Language ({esql}) provides a SQL-like interface for querying your {es} data. + +To use {esql}, this plugin needs to be installed in {ls} 8.17.4 or newer, and must be connected to {es} 8.11 or newer. + +To configure {esql} query in the plugin, set your {esql} query in the `query` parameter. + +IMPORTANT: We recommend understanding {ref}/esql-limitations.html[{esql} current limitations] before using it in production environments. + +The following is a basic {esql} query that sets the food name to transaction event based on upstream event's food ID: +[source, ruby] + filter { + elasticsearch { + hosts => [ 'https://..'] + api_key => '....' + query => ' + FROM food-index + | WHERE id == ?food_id + ' + query_params => { + "food_id" => "[food][id]" + } + } + } + +Set `config.support_escapes: true` in `logstash.yml` if you need to escape special chars in the query. + +In the result event, the plugin sets total result size in `[@metadata][total_values]` field. + +[id="{version}-plugins-{type}s-{plugin}-esql-event-mapping"] +===== Mapping {esql} result to {ls} event +{esql} returns query results in a structured tabular format, where data is organized into _columns_ (fields) and _values_ (entries). +The plugin maps each value entry to an event, populating corresponding fields. +For example, a query might produce a table like: + +[cols="2,1,1,1,2",options="header"] +|=== +|`timestamp` |`user_id` | `action` | `status.code` | `status.desc` + +|2025-04-10T12:00:00 |123 |login |200 | Success +|2025-04-10T12:05:00 |456 |purchase |403 | Forbidden (unauthorized user) +|=== + +For this case, the plugin creates two JSON look like objects as below and places them into the `target` field of the event if `target` is defined. +If `target` is not defined, the plugin places the _only_ first result at the root of the event. +[source, json] +[ + { + "timestamp": "2025-04-10T12:00:00", + "user_id": 123, + "action": "login", + "status": { + "code": 200, + "desc": "Success" + } + }, + { + "timestamp": "2025-04-10T12:05:00", + "user_id": 456, + "action": "purchase", + "status": { + "code": 403, + "desc": "Forbidden (unauthorized user)" + } + } +] + +NOTE: If your index has a mapping with sub-objects where `status.code` and `status.desc` actually dotted fields, they appear in {ls} events as a nested structure. + +[id="{version}-plugins-{type}s-{plugin}-esql-multifields"] +===== Conflict on multi-fields + +{esql} query fetches all parent and sub-fields fields if your {es} index has https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/multi-fields[multi-fields] or https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/subobjects[subobjects]. +Since {ls} events cannot contain parent field's concrete value and sub-field values together, the plugin ignores sub-fields with warning and includes parent. +We recommend using the `RENAME` (or `DROP` to avoid warning) keyword in your {esql} query explicitly rename the fields to include sub-fields into the event. + +This is a common occurrence if your template or mapping follows the pattern of always indexing strings as "text" (`field`) + " keyword" (`field.keyword`) multi-field. +In this case it's recommended to do `KEEP field` if the string is identical and there is only one subfield as the engine will optimize and retrieve the keyword, otherwise you can do `KEEP field.keyword | RENAME field.keyword as field`. + +To illustrate the situation with example, assuming your mapping has a time `time` field with `time.min` and `time.max` sub-fields as following: +[source, ruby] + "properties": { + "time": { "type": "long" }, + "time.min": { "type": "long" }, + "time.max": { "type": "long" } + } + +The {esql} result will contain all three fields but the plugin cannot map them into {ls} event. +To avoid this, you can use the `RENAME` keyword to rename the `time` parent field to get all three fields with unique fields. +[source, ruby] + ... + query => 'FROM my-index | RENAME time AS time.current' + ... + +For comprehensive ES|QL syntax reference and best practices, see the https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-syntax.html[{esql} documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["dsl", "esql"]`|No +| <<{version}-plugins-{type}s-{plugin}-query_params>> |{logstash-ref}/configuration-file-structure.html#hash[hash] or {logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"aggregation_name" => "[path][on][event]"`: + ** `aggregation_name`: aggregation name in result from {es} + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of aggregations to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"path.in.source" => "[path][on][event]"`: + ** `path.in.source`: field path in document source of result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of docinfo (`_source`) fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + * Format: `"path.in.result" => "[path][on][event]"`: + ** `path.in.result`: field path in indexed result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of indexed fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The query to be executed. +The accepted query shape is DSL query string or ES|QL. +For the DSL query string, use either `query` or `query_template`. +Read the {ref}/query-dsl-query-string-query.html[{es} query +string documentation] or {ref}/esql.html[{es} ES|QL documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-query_type"] +===== `query_type` + +* Value can be `dsl` or `esql` +* Default value is `dsl` + +Defines the <<{version}-plugins-{type}s-{plugin}-query>> shape. +When `dsl`, the query shape must be valid {es} JSON-style string. +When `esql`, the query shape must be a valid {esql} string and `index`, `query_template` and `sort` parameters are not allowed. + +[id="{version}-plugins-{type}s-{plugin}-query_params"] +===== `query_params` + +* The value type is {logstash-ref}/configuration-file-structure.html#hash[hash] or {logstash-ref}/configuration-file-structure.html#array[array]. When an array provided, the array elements are pairs of `key` and `value`. +* There is no default value for this setting + +Named parameters in {esql} to send to {es} together with <<{version}-plugins-{type}s-{plugin}-query>>. +Visit {ref}/esql-rest.html#esql-rest-params[passing parameters to query page] for more information. + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Define the target field for placing the result data. +If this setting is omitted, the target will be the root (top level) of the event. +It is highly recommended to set when using `query_type=>'esql'` to set all query results into the event. + +When `query_type=>'dsl'`, the destination fields specified in <<{version}-plugins-{type}s-{plugin}-fields>>, <<{version}-plugins-{type}s-{plugin}-aggregation_fields>>, and <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> are relative to this target. + +For example, if you want the data to be put in the `operation` field: +[source,ruby] +if [type] == "end" { + filter { + query => "type:start AND transaction:%{[transactionId]}" + elasticsearch { + target => "transaction" + fields => { + "@timestamp" => "started" + "transaction_id" => "id" + } + } + } +} + +`fields` fields will be expanded into a data structure in the `target` field, overall shape looks like this: +[source,ruby] + { + "transaction" => { + "started" => "2025-04-29T12:01:46.263Z" + "id" => "1234567890" + } + } + +NOTE: when writing to a field that already exists on the event, the previous value will be overwritten. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Filter Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +SSL + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. It can be either .jks or .p12 + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.15.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v4.0.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v4.0.0.asciidoc new file mode 100644 index 000000000..2563c5155 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v4.0.0.asciidoc @@ -0,0 +1,541 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.0 +:release_date: 2025-01-10 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v4.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> and/or <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of aggregation names to copy from elasticsearch response into Logstash event fields + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of docinfo fields to copy from old event (found via elasticsearch) into new event + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +An array of fields to copy from the old event (found via elasticsearch) into the +new event, currently being processed. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Filter Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v4.1.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v4.1.0.asciidoc new file mode 100644 index 000000000..d47fbf10e --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v4.1.0.asciidoc @@ -0,0 +1,552 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.0 +:release_date: 2025-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v4.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> and/or <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of aggregation names to copy from elasticsearch response into Logstash event fields + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of docinfo fields to copy from old event (found via elasticsearch) into new event + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +An array of fields to copy from the old event (found via elasticsearch) into the +new event, currently being processed. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Filter Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v4.1.1.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v4.1.1.asciidoc new file mode 100644 index 000000000..49c3fbc18 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v4.1.1.asciidoc @@ -0,0 +1,552 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.1 +:release_date: 2025-03-17 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v4.1.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> and/or <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of aggregation names to copy from elasticsearch response into Logstash event fields + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of docinfo fields to copy from old event (found via elasticsearch) into new event + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +An array of fields to copy from the old event (found via elasticsearch) into the +new event, currently being processed. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Filter Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v4.2.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v4.2.0.asciidoc new file mode 100644 index 000000000..761cb315a --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v4.2.0.asciidoc @@ -0,0 +1,598 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.2.0 +:release_date: 2025-05-07 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v4.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> and/or <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"aggregation_name" => "[path][on][event]"`: + ** `aggregation_name`: aggregation name in result from {es} + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of aggregations to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"path.in.source" => "[path][on][event]"`: + ** `path.in.source`: field path in document source of result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of docinfo (`_source`) fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + * Format: `"path.in.result" => "[path][on][event]"`: + ** `path.in.result`: field path in indexed result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of indexed fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. +Use either `query` or `query_template`. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Define the target field for placing the result data. +If this setting is omitted, the target will be the root (top level) of the event. + +The destination fields specified in <<{version}-plugins-{type}s-{plugin}-fields>>, <<{version}-plugins-{type}s-{plugin}-aggregation_fields>>, and <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> are relative to this target. + +For example, if you want the data to be put in the `operation` field: +[source,ruby] +if [type] == "end" { + filter { + query => "type:start AND transaction:%{[transactionId]}" + elasticsearch { + target => "transaction" + fields => { + "@timestamp" => "started" + "transaction_id" => "id" + } + } + } +} + +`fields` fields will be expanded into a data structure in the `target` field, overall shape looks like this: +[source,ruby] + { + "transaction" => { + "started" => "2025-04-29T12:01:46.263Z" + "id" => "1234567890" + } + } + +NOTE: when writing to a field that already exists on the event, the previous value will be overwritten. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Filter Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v4.3.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v4.3.0.asciidoc new file mode 100644 index 000000000..4dc26b447 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v4.3.0.asciidoc @@ -0,0 +1,724 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.3.0 +:release_date: 2025-07-21 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v4.3.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard {ls} field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> and/or <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-esql"] +==== {esql} support + +.Technical Preview +**** +The {esql} feature that allows using ES|QL queries with this plugin is in Technical Preview. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +{es} Query Language ({esql}) provides a SQL-like interface for querying your {es} data. + +To use {esql}, this plugin needs to be installed in {ls} 8.17.4 or newer, and must be connected to {es} 8.11 or newer. + +To configure {esql} query in the plugin, set your {esql} query in the `query` parameter. + +IMPORTANT: We recommend understanding {ref}/esql-limitations.html[{esql} current limitations] before using it in production environments. + +The following is a basic {esql} query that sets the food name to transaction event based on upstream event's food ID: +[source, ruby] + filter { + elasticsearch { + hosts => [ 'https://..'] + api_key => '....' + query => ' + FROM food-index + | WHERE id == ?food_id + ' + query_params => { + "food_id" => "[food][id]" + } + } + } + +Set `config.support_escapes: true` in `logstash.yml` if you need to escape special chars in the query. + +In the result event, the plugin sets total result size in `[@metadata][total_values]` field. + +[id="{version}-plugins-{type}s-{plugin}-esql-event-mapping"] +===== Mapping {esql} result to {ls} event +{esql} returns query results in a structured tabular format, where data is organized into _columns_ (fields) and _values_ (entries). +The plugin maps each value entry to an event, populating corresponding fields. +For example, a query might produce a table like: + +[cols="2,1,1,1,2",options="header"] +|=== +|`timestamp` |`user_id` | `action` | `status.code` | `status.desc` + +|2025-04-10T12:00:00 |123 |login |200 | Success +|2025-04-10T12:05:00 |456 |purchase |403 | Forbidden (unauthorized user) +|=== + +For this case, the plugin creates two JSON look like objects as below and places them into the `target` field of the event if `target` is defined. +If `target` is not defined, the plugin places the _only_ first result at the root of the event. +[source, json] +[ + { + "timestamp": "2025-04-10T12:00:00", + "user_id": 123, + "action": "login", + "status": { + "code": 200, + "desc": "Success" + } + }, + { + "timestamp": "2025-04-10T12:05:00", + "user_id": 456, + "action": "purchase", + "status": { + "code": 403, + "desc": "Forbidden (unauthorized user)" + } + } +] + +NOTE: If your index has a mapping with sub-objects where `status.code` and `status.desc` actually dotted fields, they appear in {ls} events as a nested structure. + +[id="{version}-plugins-{type}s-{plugin}-esql-multifields"] +===== Conflict on multi-fields + +{esql} query fetches all parent and sub-fields fields if your {es} index has https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/multi-fields[multi-fields] or https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/subobjects[subobjects]. +Since {ls} events cannot contain parent field's concrete value and sub-field values together, the plugin ignores sub-fields with warning and includes parent. +We recommend using the `RENAME` (or `DROP` to avoid warning) keyword in your {esql} query explicitly rename the fields to include sub-fields into the event. + +This is a common occurrence if your template or mapping follows the pattern of always indexing strings as "text" (`field`) + " keyword" (`field.keyword`) multi-field. +In this case it's recommended to do `KEEP field` if the string is identical and there is only one subfield as the engine will optimize and retrieve the keyword, otherwise you can do `KEEP field.keyword | RENAME field.keyword as field`. + +To illustrate the situation with example, assuming your mapping has a time `time` field with `time.min` and `time.max` sub-fields as following: +[source, ruby] + "properties": { + "time": { "type": "long" }, + "time.min": { "type": "long" }, + "time.max": { "type": "long" } + } + +The {esql} result will contain all three fields but the plugin cannot map them into {ls} event. +To avoid this, you can use the `RENAME` keyword to rename the `time` parent field to get all three fields with unique fields. +[source, ruby] + ... + query => 'FROM my-index | RENAME time AS time.current' + ... + +For comprehensive ES|QL syntax reference and best practices, see the https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-syntax.html[{esql} documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["dsl", "esql"]`|No +| <<{version}-plugins-{type}s-{plugin}-query_params>> |{logstash-ref}/configuration-file-structure.html#hash[hash] or {logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"aggregation_name" => "[path][on][event]"`: + ** `aggregation_name`: aggregation name in result from {es} + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of aggregations to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + * Format: `"path.in.source" => "[path][on][event]"`: + ** `path.in.source`: field path in document source of result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of docinfo (`_source`) fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + * Format: `"path.in.result" => "[path][on][event]"`: + ** `path.in.result`: field path in indexed result from {es}, using dot-notation + ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation + +A mapping of indexed fields to copy into the <<{version}-plugins-{type}s-{plugin}-target>> of the current event. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The query to be executed. +The accepted query shape is DSL query string or ES|QL. +For the DSL query string, use either `query` or `query_template`. +Read the {ref}/query-dsl-query-string-query.html[{es} query +string documentation] or {ref}/esql.html[{es} ES|QL documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-query_type"] +===== `query_type` + +* Value can be `dsl` or `esql` +* Default value is `dsl` + +Defines the <<{version}-plugins-{type}s-{plugin}-query>> shape. +When `dsl`, the query shape must be valid {es} JSON-style string. +When `esql`, the query shape must be a valid {esql} string and `index`, `query_template` and `sort` parameters are not allowed. + +[id="{version}-plugins-{type}s-{plugin}-query_params"] +===== `query_params` + +* The value type is {logstash-ref}/configuration-file-structure.html#hash[hash] or {logstash-ref}/configuration-file-structure.html#array[array]. When an array provided, the array elements are pairs of `key` and `value`. +* There is no default value for this setting + +Named parameters in {esql} to send to {es} together with <<{version}-plugins-{type}s-{plugin}-query>>. +Visit {ref}/esql-rest.html#esql-rest-params[passing parameters to query page] for more information. + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. +Use either `query` or `query_template`. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Define the target field for placing the result data. +If this setting is omitted, the target will be the root (top level) of the event. +It is highly recommended to set when using `query_type=>'esql'` to set all query results into the event. + +When `query_type=>'dsl'`, the destination fields specified in <<{version}-plugins-{type}s-{plugin}-fields>>, <<{version}-plugins-{type}s-{plugin}-aggregation_fields>>, and <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> are relative to this target. + +For example, if you want the data to be put in the `operation` field: +[source,ruby] +if [type] == "end" { + filter { + query => "type:start AND transaction:%{[transactionId]}" + elasticsearch { + target => "transaction" + fields => { + "@timestamp" => "started" + "transaction_id" => "id" + } + } + } +} + +`fields` fields will be expanded into a data structure in the `target` field, overall shape looks like this: +[source,ruby] + { + "transaction" => { + "started" => "2025-04-29T12:01:46.263Z" + "id" => "1234567890" + } + } + +NOTE: when writing to a field that already exists on the event, the previous value will be overwritten. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Filter Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/fingerprint-index.asciidoc b/docs/versioned-plugins/filters/fingerprint-index.asciidoc index 819fd59df..4186b4129 100644 --- a/docs/versioned-plugins/filters/fingerprint-index.asciidoc +++ b/docs/versioned-plugins/filters/fingerprint-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-03-19 | <> | 2023-05-12 | <> | 2023-01-13 | <> | 2022-06-16 @@ -23,6 +24,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::fingerprint-v3.4.4.asciidoc[] include::fingerprint-v3.4.3.asciidoc[] include::fingerprint-v3.4.2.asciidoc[] include::fingerprint-v3.4.1.asciidoc[] diff --git a/docs/versioned-plugins/filters/fingerprint-v3.4.4.asciidoc b/docs/versioned-plugins/filters/fingerprint-v3.4.4.asciidoc new file mode 100644 index 000000000..2dd19dfe9 --- /dev/null +++ b/docs/versioned-plugins/filters/fingerprint-v3.4.4.asciidoc @@ -0,0 +1,223 @@ +:plugin: fingerprint +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.4.4 +:release_date: 2024-03-19 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-fingerprint/blob/v3.4.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Fingerprint filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Create consistent hashes (fingerprints) of one or more fields and store +the result in a new field. + +You can use this plugin to create consistent document ids when events are +inserted into Elasticsearch. This approach means that existing documents can be +updated instead of creating new documents. + +NOTE: When the `method` option is set to `UUID` the result won't be +a consistent hash but a random +https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID]. +To generate UUIDs, prefer the {logstash-ref}/plugins-filters-uuid.html[uuid filter]. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +This plugin adds a hash value to event as an identifier. You can configure the `target` option to change the output field. + +When ECS compatibility is disabled, the hash value is stored in the `fingerprint` field. +When ECS is enabled, the value is stored in the `[event][hash]` field. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-base64encode>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-concatenate_sources>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-concatenate_all_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-method>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "MURMUR3_128", IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-base64encode"] +===== `base64encode` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint methods will produce +base64 encoded rather than hex encoded strings. + +[id="{version}-plugins-{type}s-{plugin}-concatenate_sources"] +===== `concatenate_sources` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true` and `method` isn't `UUID` or `PUNCTUATION`, the +plugin concatenates the names and values of all fields given in the +`source` option into one string (like the old checksum filter) before +doing the fingerprint computation. + +If `false` and multiple source fields are given, the target field will be single +fingerprint of the last source field. + +**Example: `concatenate_sources`=false** + +This example produces a single fingerprint that is computed from "birthday," the +last source field. + +[source,ruby] +----- +fingerprint { + source => ["user_id", "siblings", "birthday"] +} +----- + +The output is: + +[source,ruby] +----- +"fingerprint" => "6b6390a4416131f82b6ffb509f6e779e5dd9630f". +----- + +**Example: `concatenate_sources`=false with array** + +If the last source field is an array, you get an array of fingerprints. + +In this example, "siblings" is an array ["big brother", "little sister", "little brother"]. + +[source,ruby] +----- +fingerprint { + source => ["user_id", "siblings"] +} +----- + +The output is: + +[source,ruby] +----- + "fingerprint" => [ + [0] "8a8a9323677f4095fcf0c8c30b091a0133b00641", + [1] "2ce11b313402e0e9884e094409f8d9fcf01337c2", + [2] "adc0b90f9391a82098c7b99e66a816e9619ad0a7" + ], +----- + +[id="{version}-plugins-{type}s-{plugin}-concatenate_all_fields"] +===== `concatenate_all_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true` and `method` isn't `UUID` or `PUNCTUATION`, the +plugin concatenates the names and values of all fields of the event +into one string (like the old checksum filter) before doing the +fingerprint computation. If `false` and at least one source field is +given, the target field will be an array with fingerprints of the +source fields given. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: unstructured data added at root level +** `v1`: uses `[event][hash]` fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-key"] +===== `key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +When used with the `IPV4_NETWORK` method fill in the subnet prefix length. +With other methods, optionally fill in the HMAC key. + +[id="{version}-plugins-{type}s-{plugin}-method"] +===== `method` + + * This is a required setting. + * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`, `MURMUR3`, `MURMUR3_128`, `IPV4_NETWORK`, `UUID`, `PUNCTUATION` + * Default value is `"SHA1"` + +The fingerprint method to use. + +If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set, the +corresponding cryptographic hash function and the keyed-hash (HMAC) digest function +are used to generate the fingerprint. + +If set to `MURMUR3` or `MURMUR3_128` the non-cryptographic MurmurHash function (either the 32-bit or 128-bit implementation, respectively) will be used. + +If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and +the hash value will be the masked-out address using the number of bits +specified in the `key` option. For example, with "1.2.3.4" as the input +and `key` set to 16, the hash becomes "1.2.0.0". + +If set to `PUNCTUATION`, all non-punctuation characters will be removed +from the input string. + +If set to `UUID`, a +https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID] will +be generated. The result will be random and thus not a consistent hash. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `"message"` + +The name(s) of the source field(s) whose contents will be used +to create the fingerprint. If an array is given, see the +`concatenate_sources` option. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"fingerprint"` when ECS is disabled + * Default value is `"[event][hash]"` when ECS is enabled + +The name of the field where the generated fingerprint will be stored. +Any current contents of that field will be overwritten. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/geoip-index.asciidoc b/docs/versioned-plugins/filters/geoip-index.asciidoc index d98aab3b6..38b3ce3fe 100644 --- a/docs/versioned-plugins/filters/geoip-index.asciidoc +++ b/docs/versioned-plugins/filters/geoip-index.asciidoc @@ -5,6 +5,8 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-10-11 +| <> | 2024-05-22 | <> | 2023-02-07 | <> | 2022-03-18 | <> | 2022-01-26 @@ -42,6 +44,8 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-15 |======================================================================= +include::geoip-v7.3.1.asciidoc[] +include::geoip-v7.3.0.asciidoc[] include::geoip-v7.2.13.asciidoc[] include::geoip-v7.2.12.asciidoc[] include::geoip-v7.2.11.asciidoc[] diff --git a/docs/versioned-plugins/filters/geoip-v7.3.0.asciidoc b/docs/versioned-plugins/filters/geoip-v7.3.0.asciidoc new file mode 100644 index 000000000..5f8fec69f --- /dev/null +++ b/docs/versioned-plugins/filters/geoip-v7.3.0.asciidoc @@ -0,0 +1,399 @@ +:plugin: geoip +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.3.0 +:release_date: 2024-05-22 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-geoip/blob/v7.3.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Geoip filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The GeoIP filter adds information about the geographical location of IP addresses, +based on data from the MaxMind GeoLite2 databases. + +==== Supported Databases + +This plugin is bundled with https://dev.maxmind.com/geoip/geoip2/geolite2[GeoLite2] City database out of the box. From MaxMind's description -- +"GeoLite2 databases are free IP geolocation databases comparable to, but less accurate than, MaxMind’s +GeoIP2 databases". Please see GeoIP Lite2 license for more details. + +https://www.maxmind.com/en/geoip2-databases[Commercial databases] from MaxMind are also supported in this plugin. + +If you need to use databases other than the bundled GeoLite2 City, you can download them directly +from MaxMind's website and use the `database` option to specify their location. The GeoLite2 databases +can be https://dev.maxmind.com/geoip/geoip2/geolite2[downloaded from here]. + +If you would like to get Autonomous System Number(ASN) information, you can use the GeoLite2-ASN database. + +[id="{version}-plugins-{type}s-{plugin}-database_license"] +==== Database License + +https://www.maxmind.com[MaxMind] changed from releasing the GeoIP database under +a Creative Commons (CC) license to a proprietary end-user license agreement +(EULA). The MaxMind EULA requires Logstash to update the MaxMind database +within 30 days of a database update. + +The GeoIP filter plugin can manage the database for users running the Logstash default +distribution, or you can manage +database updates on your own. The behavior is controlled by the `database` setting and by the auto-update feature. +When you use the default `database` setting and the auto-update feature is enabled, Logstash ensures that the plugin is +using the latest version of the database. +Otherwise, you are responsible for maintaining compliance. + +The Logstash open source distribution uses the MaxMind Creative Commons license +database by default. + +[id="{version}-plugins-{type}s-{plugin}-database_auto"] +==== Database Auto-update + +This plugin bundles Creative Commons (CC) license databases. +If the auto-update feature is enabled in `logstash.yml`(as it is by default), Logstash checks for database updates every day. It downloads the latest and can replace the old database +while the plugin is running. + +NOTE: If the auto-update feature is disabled or the database has never been updated successfully, as in air-gapped environments, Logstash can use CC license databases indefinitely. + +After Logstash has switched to a EULA licensed database, the geoip filter will +stop enriching events in order to maintain compliance if Logstash fails to +check for database updates for 30 days. +Events will be tagged with `_geoip_expired_database` tag to facilitate the handling of this situation. + +NOTE: If the auto-update feature is enabled, Logstash upgrades from the CC database license to the EULA version on the first download. + +TIP: When possible, allow Logstash to access the internet to download databases so that they are always up-to-date. + +**Disable the auto-update feature** + +If you work in air-gapped environment and want to disable the database auto-update feature, set the `xpack.geoip.downloader.enabled` value to `false` in `logstash.yml`. + +When the auto-update feature is disabled, Logstash uses the Creative Commons (CC) license databases indefinitely, and any previously downloaded version of the EULA databases will be deleted. + +[id="{version}-plugins-{type}s-{plugin}-manage_update"] +==== Manage your own database updates + +**Use an HTTP proxy** + +If you can't connect directly to the Elastic GeoIP endpoint, consider setting up +an HTTP proxy server. You can then specify the proxy with `http_proxy` environment variable. + +[source,sh] +---- +export http_proxy="http://PROXY_IP:PROXY_PORT" +---- + +**Use a custom endpoint (air-gapped environments)** + +If you work in air-gapped environment and can't update your databases from the Elastic endpoint, +You can then download databases from MaxMind and bootstrap the service. + +. Download both `GeoLite2-ASN.mmdb` and `GeoLite2-City.mmdb` database files from the +http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site]. + +. Copy both database files to a single directory. + +. https://www.elastic.co/downloads/elasticsearch[Download {es}]. + +. From your {es} directory, run: ++ +[source,sh] +---- +./bin/elasticsearch-geoip -s my/database/dir +---- + +. Serve the static database files from your directory. For example, you can use +Docker to serve the files from nginx server: ++ +[source,sh] +---- +docker run -p 8080:80 -v my/database/dir:/usr/share/nginx/html:ro nginx +---- + +. Specify the service's endpoint URL using the +`xpack.geoip.download.endpoint=http://localhost:8080/overview.json` setting in `logstash.yml`. + +Logstash gets automatic updates from this service. + +[id="{version}-plugins-{type}s-{plugin}-metrics"] +==== Database Metrics + +You can monitor database status through the {logstash-ref}/node-stats-api.html#node-stats-api[Node Stats API]. + +The following request returns a JSON document containing database manager stats, +including: + +* database status and freshness +** `geoip_download_manager.database.*.status` +*** `init` : initial CC database status +*** `up_to_date` : using up-to-date EULA database +*** `to_be_expired` : 25 days without calling service +*** `expired` : 30 days without calling service +** `fail_check_in_days` : number of days Logstash fails to call service since the last success +* info about download successes and failures +** `geoip_download_manager.download_stats.successes` number of successful checks and downloads +** `geoip_download_manager.download_stats.failures` number of failed check or download +** `geoip_download_manager.download_stats.status` +*** `updating` : check and download at the moment +*** `succeeded` : last download succeed +*** `failed` : last download failed + +[source,js] +-------------------------------------------------- +curl -XGET 'localhost:9600/_node/stats/geoip_download_manager?pretty' +-------------------------------------------------- + +Example response: + +[source,js] +-------------------------------------------------- +{ + "geoip_download_manager" : { + "database" : { + "ASN" : { + "status" : "up_to_date", + "fail_check_in_days" : 0, + "last_updated_at": "2021-06-21T16:06:54+02:00" + }, + "City" : { + "status" : "up_to_date", + "fail_check_in_days" : 0, + "last_updated_at": "2021-06-21T16:06:54+02:00" + } + }, + "download_stats" : { + "successes" : 15, + "failures" : 1, + "last_checked_at" : "2021-06-21T16:07:03+02:00", + "status" : "succeeded" + } + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-field-mapping"] +==== Field mapping + +When this plugin is run with <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> disabled, the MaxMind DB's fields are added directly to the <<{version}-plugins-{type}s-{plugin}-target>>. +When ECS compatibility is enabled, the fields are structured to fit into an ECS shape. + +[cols="3,5,3"] +|=========================== +| Database Field Name | ECS Field | Example + +| `ip` | `[ip]` | `12.34.56.78` +| `anonymous` | `[ip_traits][anonymous]` | `false` +| `anonymous_vpn` | `[ip_traits][anonymous_vpn]` | `false` +| `hosting_provider` | `[ip_traits][hosting_provider]` | `true` +| `network` | `[ip_traits][network]` | `12.34.56.78/20` +| `public_proxy` | `[ip_traits][public_proxy]` | `true` +| `residential_proxy` | `[ip_traits][residential_proxy]` | `false` +| `tor_exit_node` | `[ip_traits][tor_exit_node]` | `true` + +| `city_name` | `[geo][city_name]` | `Seattle` +| `country_name` | `[geo][country_name]` | `United States` +| `continent_code` | `[geo][continent_code]` | `NA` +| `continent_name` | `[geo][continent_name]` | `North America` +| `country_code2` | `[geo][country_iso_code]` | `US` +| `country_code3` | _N/A_ | `US` + + _maintained for legacy + support, but populated + with 2-character country + code_ + +| `postal_code` | `[geo][postal_code]` | `98106` +| `region_name` | `[geo][region_name]` | `Washington` +| `region_code` | `[geo][region_code]` | `WA` +| `region_iso_code`* | `[geo][region_iso_code]` | `US-WA` +| `timezone` | `[geo][timezone]` | `America/Los_Angeles` +| `location`* | `[geo][location]` | `{"lat": 47.6062, "lon": -122.3321}"` +| `latitude` | `[geo][location][lat]` | `47.6062` +| `longitude` | `[geo][location][lon]` | `-122.3321` + +| `domain` | `[domain]` | `example.com` + +| `asn` | `[as][number]` | `98765` +| `as_org` | `[as][organization][name]` | `Elastic, NV` + +| `isp` | `[mmdb][isp]` | `InterLink Supra LLC` +| `dma_code` | `[mmdb][dma_code]` | `819` +| `organization` | `[mmdb][organization]` | `Elastic, NV` +|=========================== + +NOTE: `*` indicates a composite field, which is only populated if GeoIP lookup result contains all components. + +==== Details + +When using a City database, the enrichment is aborted if no latitude/longitude pair is available. + +The `location` field combines the latitude and longitude into a structure called https://datatracker.ietf.org/doc/html/rfc7946[GeoJSON]. +When you are using a default <<{version}-plugins-{type}s-{plugin}-target>>, the templates provided by the {logstash-ref}/plugins-outputs-elasticsearch.html[elasticsearch output] map the field to an {ref}/geo-point.html[Elasticsearch Geo_point datatype]. + +As this field is a `geo_point` _and_ it is still valid GeoJSON, you get +the awesomeness of Elasticsearch's geospatial query, facet and filter functions +and the flexibility of having GeoJSON for all other applications (like Kibana's +map visualization). + +[NOTE] +-- +This product includes GeoLite2 data created by MaxMind, available from +http://www.maxmind.com. This database is licensed under +http://creativecommons.org/licenses/by-sa/4.0/[Creative Commons Attribution-ShareAlike 4.0 International License]. + +Versions 4.0.0 and later of the GeoIP filter use the MaxMind GeoLite2 database +and support both IPv4 and IPv6 lookups. Versions prior to 4.0.0 use the legacy +MaxMind GeoLite database and support IPv4 lookups only. +-- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Geoip Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-database>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-default_database_type>> |`City` or `ASN`|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +GeoIP lookup is surprisingly expensive. This filter uses an cache to take advantage of the fact that +IPs agents are often found adjacent to one another in log files and rarely have a random distribution. +The higher you set this the more likely an item is to be in the cache and the faster this filter will run. +However, if you set this too high you can use more memory than desired. +Since the Geoip API upgraded to v2, there is not any eviction policy so far, if cache is full, no more record can be added. +Experiment with different values for this option to find the best performance for your dataset. + +This MUST be set to a value > 0. There is really no reason to not want this behavior, the overhead is minimal +and the speed gains are large. + +It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter +of the same geoip_type share the same cache. The last declared cache size will 'win'. The reason for this is that there would be no benefit +to having multiple caches for different instances at different points in the pipeline, that would just increase the +number of cache misses and waste memory. + +[id="{version}-plugins-{type}s-{plugin}-database"] +===== `database` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * If not specified, the database defaults to the `GeoLite2 City` database that ships with Logstash. + +The path to MaxMind's database file that Logstash should use. +The default database is `GeoLite2-City`. +This plugin supports several free databases (`GeoLite2-City`, `GeoLite2-Country`, `GeoLite2-ASN`) +and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`, `GeoIP2-Domain`, `GeoIP2-Enterprise`, `GeoIP2-Anonymous-IP`). + +Database auto-update applies to the default distribution. +When `database` points to user's database path, auto-update is disabled. +See <<{version}-plugins-{type}s-{plugin}-database_license,Database License>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-default_database_type"] +===== `default_database_type` + +This plugin now includes both the GeoLite2-City and GeoLite2-ASN databases. If `database` and `default_database_type` are unset, the GeoLite2-City database will be selected. To use the included GeoLite2-ASN database, set `default_database_type` to `ASN`. + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value is `City` + * The only acceptable values are `City` and `ASN` + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +An array of geoip fields to be included in the event. + +Possible fields depend on the database type. +By default, all geoip fields from the relevant database are included in the event. + +For a complete list of available fields and how they map to an event's structure, see <<{version}-plugins-{type}s-{plugin}-field-mapping,field mapping>>. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: unstructured geo data added at root level +** `v1`, `v8`: use fields that are compatible with Elastic Common Schema. Example: `[client][geo][country_name]`. See <<{version}-plugins-{type}s-{plugin}-field-mapping,field mapping>> for more info. +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-target>>. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The field containing the IP address or hostname to map via geoip. If +this field is an array, only the first value will be used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_geoip_lookup_failure"]` + +Tags the event on failure to look up geo information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is an optional setting with condition. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `geoip` + ** ECS Compatibility enabled: If `source` is an `ip` sub-field, eg. `[client][ip]`, + `target` will automatically set to the parent field, in this example `client`, + otherwise, `target` is a required setting + *** `geo` field is nested in `[client][geo]` + *** ECS compatible values are `client`, `destination`, `host`, `observer`, `server`, `source` + +Specify the field into which Logstash should store the geoip data. +This can be useful, for example, if you have `src_ip` and `dst_ip` fields and +would like the GeoIP information of both IPs. + +If you save the data to a target field other than `geoip` and want to use the +`geo_point` related functions in Elasticsearch, you need to alter the template +provided with the Elasticsearch output and configure the output to use the +new template. + +Even if you don't use the `geo_point` mapping, the `[target][location]` field +is still valid GeoJSON. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/geoip-v7.3.1.asciidoc b/docs/versioned-plugins/filters/geoip-v7.3.1.asciidoc new file mode 100644 index 000000000..ba3ebe4a2 --- /dev/null +++ b/docs/versioned-plugins/filters/geoip-v7.3.1.asciidoc @@ -0,0 +1,399 @@ +:plugin: geoip +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.3.1 +:release_date: 2024-10-11 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-geoip/blob/v7.3.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Geoip filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The GeoIP filter adds information about the geographical location of IP addresses, +based on data from the MaxMind GeoLite2 databases. + +==== Supported Databases + +This plugin is bundled with https://dev.maxmind.com/geoip/geoip2/geolite2[GeoLite2] City database out of the box. From MaxMind's description -- +"GeoLite2 databases are free IP geolocation databases comparable to, but less accurate than, MaxMind’s +GeoIP2 databases". Please see GeoIP Lite2 license for more details. + +https://www.maxmind.com/en/geoip2-databases[Commercial databases] from MaxMind are also supported in this plugin. + +If you need to use databases other than the bundled GeoLite2 City, you can download them directly +from MaxMind's website and use the `database` option to specify their location. The GeoLite2 databases +can be https://dev.maxmind.com/geoip/geoip2/geolite2[downloaded from here]. + +If you would like to get Autonomous System Number(ASN) information, you can use the GeoLite2-ASN database. + +[id="{version}-plugins-{type}s-{plugin}-database_license"] +==== Database License + +https://www.maxmind.com[MaxMind] changed from releasing the GeoIP database under +a Creative Commons (CC) license to a proprietary end-user license agreement +(EULA). The MaxMind EULA requires Logstash to update the MaxMind database +within 30 days of a database update. + +The GeoIP filter plugin can manage the database for users running the Logstash default +distribution, or you can manage +database updates on your own. The behavior is controlled by the `database` setting and by the auto-update feature. +When you use the default `database` setting and the auto-update feature is enabled, Logstash ensures that the plugin is +using the latest version of the database. +Otherwise, you are responsible for maintaining compliance. + +The Logstash open source distribution uses the MaxMind Creative Commons license +database by default. + +[id="{version}-plugins-{type}s-{plugin}-database_auto"] +==== Database Auto-update + +This plugin bundles Creative Commons (CC) license databases. +If the auto-update feature is enabled in `logstash.yml`(as it is by default), Logstash checks for database updates every day. It downloads the latest and can replace the old database +while the plugin is running. + +NOTE: If the auto-update feature is disabled or the database has never been updated successfully, as in air-gapped environments, Logstash can use CC license databases indefinitely. + +After Logstash has switched to a EULA licensed database, the geoip filter will +stop enriching events in order to maintain compliance if Logstash fails to +check for database updates for 30 days. +Events will be tagged with `_geoip_expired_database` tag to facilitate the handling of this situation. + +NOTE: If the auto-update feature is enabled, Logstash upgrades from the CC database license to the EULA version on the first download. + +TIP: When possible, allow Logstash to access the internet to download databases so that they are always up-to-date. + +**Disable the auto-update feature** + +If you work in air-gapped environment and want to disable the database auto-update feature, set the `xpack.geoip.downloader.enabled` value to `false` in `logstash.yml`. + +When the auto-update feature is disabled, Logstash uses the Creative Commons (CC) license databases indefinitely, and any previously downloaded version of the EULA databases will be deleted. + +[id="{version}-plugins-{type}s-{plugin}-manage_update"] +==== Manage your own database updates + +**Use an HTTP proxy** + +If you can't connect directly to the Elastic GeoIP endpoint, consider setting up +an HTTP proxy server. You can then specify the proxy with `http_proxy` environment variable. + +[source,sh] +---- +export http_proxy="http://PROXY_IP:PROXY_PORT" +---- + +**Use a custom endpoint (air-gapped environments)** + +If you work in air-gapped environment and can't update your databases from the Elastic endpoint, +You can then download databases from MaxMind and bootstrap the service. + +. Download both `GeoLite2-ASN.mmdb` and `GeoLite2-City.mmdb` database files from the +http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site]. + +. Copy both database files to a single directory. + +. https://www.elastic.co/downloads/elasticsearch[Download {es}]. + +. From your {es} directory, run: ++ +[source,sh] +---- +./bin/elasticsearch-geoip -s my/database/dir +---- + +. Serve the static database files from your directory. For example, you can use +Docker to serve the files from nginx server: ++ +[source,sh] +---- +docker run -p 8080:80 -v my/database/dir:/usr/share/nginx/html:ro nginx +---- + +. Specify the service's endpoint URL using the +`xpack.geoip.download.endpoint=http://localhost:8080/overview.json` setting in `logstash.yml`. + +Logstash gets automatic updates from this service. + +[id="{version}-plugins-{type}s-{plugin}-metrics"] +==== Database Metrics + +You can monitor database status through the {logstash-ref}/node-stats-api.html#node-stats-api[Node Stats API]. + +The following request returns a JSON document containing database manager stats, +including: + +* database status and freshness +** `geoip_download_manager.database.*.status` +*** `init` : initial CC database status +*** `up_to_date` : using up-to-date EULA database +*** `to_be_expired` : 25 days without calling service +*** `expired` : 30 days without calling service +** `fail_check_in_days` : number of days Logstash fails to call service since the last success +* info about download successes and failures +** `geoip_download_manager.download_stats.successes` number of successful checks and downloads +** `geoip_download_manager.download_stats.failures` number of failed check or download +** `geoip_download_manager.download_stats.status` +*** `updating` : check and download at the moment +*** `succeeded` : last download succeed +*** `failed` : last download failed + +[source,js] +-------------------------------------------------- +curl -XGET 'localhost:9600/_node/stats/geoip_download_manager?pretty' +-------------------------------------------------- + +Example response: + +[source,js] +-------------------------------------------------- +{ + "geoip_download_manager" : { + "database" : { + "ASN" : { + "status" : "up_to_date", + "fail_check_in_days" : 0, + "last_updated_at": "2021-06-21T16:06:54+02:00" + }, + "City" : { + "status" : "up_to_date", + "fail_check_in_days" : 0, + "last_updated_at": "2021-06-21T16:06:54+02:00" + } + }, + "download_stats" : { + "successes" : 15, + "failures" : 1, + "last_checked_at" : "2021-06-21T16:07:03+02:00", + "status" : "succeeded" + } + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-field-mapping"] +==== Field mapping + +When this plugin is run with <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> disabled, the MaxMind DB's fields are added directly to the <<{version}-plugins-{type}s-{plugin}-target>>. +When ECS compatibility is enabled, the fields are structured to fit into an ECS shape. + +[cols="3,5,3"] +|=========================== +| Database Field Name | ECS Field | Example + +| `ip` | `[ip]` | `12.34.56.78` +| `anonymous` | `[ip_traits][anonymous]` | `false` +| `anonymous_vpn` | `[ip_traits][anonymous_vpn]` | `false` +| `hosting_provider` | `[ip_traits][hosting_provider]` | `true` +| `network` | `[ip_traits][network]` | `12.34.56.78/20` +| `public_proxy` | `[ip_traits][public_proxy]` | `true` +| `residential_proxy` | `[ip_traits][residential_proxy]` | `false` +| `tor_exit_node` | `[ip_traits][tor_exit_node]` | `true` + +| `city_name` | `[geo][city_name]` | `Seattle` +| `country_name` | `[geo][country_name]` | `United States` +| `continent_code` | `[geo][continent_code]` | `NA` +| `continent_name` | `[geo][continent_name]` | `North America` +| `country_code2` | `[geo][country_iso_code]` | `US` +| `country_code3` | _N/A_ | `US` + + _maintained for legacy + support, but populated + with 2-character country + code_ + +| `postal_code` | `[geo][postal_code]` | `98106` +| `region_name` | `[geo][region_name]` | `Washington` +| `region_code` | `[geo][region_code]` | `WA` +| `region_iso_code`* | `[geo][region_iso_code]` | `US-WA` +| `timezone` | `[geo][timezone]` | `America/Los_Angeles` +| `location`* | `[geo][location]` | `{"lat": 47.6062, "lon": -122.3321}"` +| `latitude` | `[geo][location][lat]` | `47.6062` +| `longitude` | `[geo][location][lon]` | `-122.3321` + +| `domain` | `[domain]` | `example.com` + +| `asn` | `[as][number]` | `98765` +| `as_org` | `[as][organization][name]` | `Elastic, NV` + +| `isp` | `[mmdb][isp]` | `InterLink Supra LLC` +| `dma_code` | `[mmdb][dma_code]` | `819` +| `organization` | `[mmdb][organization]` | `Elastic, NV` +|=========================== + +NOTE: `*` indicates a composite field, which is only populated if GeoIP lookup result contains all components. + +==== Details + +When using a City database, the enrichment is aborted if no latitude/longitude pair is available. + +The `location` field combines the latitude and longitude into a structure called https://datatracker.ietf.org/doc/html/rfc7946[GeoJSON]. +When you are using a default <<{version}-plugins-{type}s-{plugin}-target>>, the templates provided by the {logstash-ref}/plugins-outputs-elasticsearch.html[elasticsearch output] map the field to an {ref}/geo-point.html[Elasticsearch Geo_point datatype]. + +As this field is a `geo_point` _and_ it is still valid GeoJSON, you get +the awesomeness of Elasticsearch's geospatial query, facet and filter functions +and the flexibility of having GeoJSON for all other applications (like Kibana's +map visualization). + +[NOTE] +-- +This product includes GeoLite2 data created by MaxMind, available from +http://www.maxmind.com. This database is licensed under +http://creativecommons.org/licenses/by-sa/4.0/[Creative Commons Attribution-ShareAlike 4.0 International License]. + +Versions 4.0.0 and later of the GeoIP filter use the MaxMind GeoLite2 database +and support both IPv4 and IPv6 lookups. Versions prior to 4.0.0 use the legacy +MaxMind GeoLite database and support IPv4 lookups only. +-- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Geoip Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-database>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-default_database_type>> |`City` or `ASN`|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +GeoIP lookup is surprisingly expensive. This filter uses an cache to take advantage of the fact that +IPs agents are often found adjacent to one another in log files and rarely have a random distribution. +The higher you set this the more likely an item is to be in the cache and the faster this filter will run. +However, if you set this too high you can use more memory than desired. +Since the Geoip API upgraded to v2, there is not any eviction policy so far, if cache is full, no more record can be added. +Experiment with different values for this option to find the best performance for your dataset. + +This MUST be set to a value > 0. There is really no reason to not want this behavior, the overhead is minimal +and the speed gains are large. + +It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter +of the same geoip_type share the same cache. The last declared cache size will 'win'. The reason for this is that there would be no benefit +to having multiple caches for different instances at different points in the pipeline, that would just increase the +number of cache misses and waste memory. + +[id="{version}-plugins-{type}s-{plugin}-database"] +===== `database` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * If not specified, the database defaults to the `GeoLite2 City` database that ships with Logstash. + +The path to MaxMind's database file that Logstash should use. +The default database is `GeoLite2-City`. +This plugin supports several free databases (`GeoLite2-City`, `GeoLite2-Country`, `GeoLite2-ASN`) +and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`, `GeoIP2-Domain`, `GeoIP2-Enterprise`, `GeoIP2-Anonymous-IP`). + +Database auto-update applies to the default distribution. +When `database` points to user's database path, auto-update is disabled. +See <<{version}-plugins-{type}s-{plugin}-database_license,Database License>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-default_database_type"] +===== `default_database_type` + +This plugin now includes both the GeoLite2-City and GeoLite2-ASN databases. If `database` and `default_database_type` are unset, the GeoLite2-City database will be selected. To use the included GeoLite2-ASN database, set `default_database_type` to `ASN`. + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value is `City` + * The only acceptable values are `City` and `ASN` + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +An array of geoip fields to be included in the event. + +Possible fields depend on the database type. +By default, all geoip fields from the relevant database are included in the event. + +For a complete list of available fields and how they map to an event's structure, see <<{version}-plugins-{type}s-{plugin}-field-mapping,field mapping>>. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: unstructured geo data added at root level +** `v1`, `v8`: use fields that are compatible with Elastic Common Schema. Example: `[client][geo][country_name]`. See <<{version}-plugins-{type}s-{plugin}-field-mapping,field mapping>> for more info. +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-target>>. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The field containing the IP address or hostname to map via geoip. If +this field is an array, only the first value will be used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_geoip_lookup_failure"]` + +Tags the event on failure to look up geo information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is an optional setting with condition. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `geoip` + ** ECS Compatibility enabled: If `source` is an `ip` sub-field, eg. `[client][ip]`, + `target` will automatically set to the parent field, in this example `client`, + otherwise, `target` is a required setting + *** `geo` field is nested in `[client][geo]` + *** ECS compatible values are `client`, `destination`, `host`, `observer`, `server`, `source` + +Specify the field into which Logstash should store the geoip data. +This can be useful, for example, if you have `src_ip` and `dst_ip` fields and +would like the GeoIP information of both IPs. + +If you save the data to a target field other than `geoip` and want to use the +`geo_point` related functions in Elasticsearch, you need to alter the template +provided with the Elasticsearch output and configure the output to use the +new template. + +Even if you don't use the `geo_point` mapping, the `[target][location]` field +is still valid GeoJSON. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/http-index.asciidoc b/docs/versioned-plugins/filters/http-index.asciidoc index eb4da532b..19b689046 100644 --- a/docs/versioned-plugins/filters/http-index.asciidoc +++ b/docs/versioned-plugins/filters/http-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-12-18 +| <> | 2024-06-19 +| <> | 2024-01-11 | <> | 2023-09-01 | <> | 2023-03-21 | <> | 2023-03-15 @@ -20,6 +23,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2018-12-12 |======================================================================= +include::http-v2.0.0.asciidoc[] +include::http-v1.6.0.asciidoc[] +include::http-v1.5.1.asciidoc[] include::http-v1.5.0.asciidoc[] include::http-v1.4.3.asciidoc[] include::http-v1.4.2.asciidoc[] diff --git a/docs/versioned-plugins/filters/http-v1.5.1.asciidoc b/docs/versioned-plugins/filters/http-v1.5.1.asciidoc new file mode 100644 index 000000000..1c59509c1 --- /dev/null +++ b/docs/versioned-plugins/filters/http-v1.5.1.asciidoc @@ -0,0 +1,531 @@ +:plugin: http +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.5.1 +:release_date: 2024-01-11 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-http/blob/v1.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== HTTP filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The HTTP filter provides integration with external web services/REST APIs. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +The plugin includes sensible defaults that change based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>. +When targeting an ECS version, headers are set as `@metadata` and the `target_body` is a required option. +See <<{version}-plugins-{type}s-{plugin}-target_body>>, and <<{version}-plugins-{type}s-{plugin}-target_headers>>. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== HTTP Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-body>> | String, Array or Hash |No +| <<{version}-plugins-{type}s-{plugin}-body_format>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-target_body>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target_headers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-verb>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +There are also multiple configuration options related to the HTTP connectivity: + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|no +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-body"] +===== `body` + + * Value type can be a {logstash-ref}/configuration-file-structure.html#string[string], {logstash-ref}/configuration-file-structure.html#number[number], {logstash-ref}/configuration-file-structure.html#array[array] or {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +The body of the HTTP request to be sent. + +An example to send `body` as json +``` +http { + body => { + "key1" => "constant_value" + "key2" => "%{[field][reference]}" + } + body_format => "json" +} +``` + +[id="{version}-plugins-{type}s-{plugin}-body_format"] +===== `body_format` + + * Value type can be either `"json"` or `"text"` + * Default value is `"text"` + +If set to `"json"` and the <<{version}-plugins-{type}s-{plugin}-body>> is a type of {logstash-ref}/configuration-file-structure.html#array[array] or {logstash-ref}/configuration-file-structure.html#hash[hash], the body will be serialized as JSON. Otherwise it is sent as is. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (for example, response headers target `headers` field by default) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, headers are added as metadata) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-target_body>> and +<<{version}-plugins-{type}s-{plugin}-target_headers>>. + + +[id="{version}-plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +The HTTP headers to be sent in the request. Both the names of the headers and their +values can reference values from event fields. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +Define the query string parameters (key-value pairs) to be sent in the HTTP request. + +[id="{version}-plugins-{type}s-{plugin}-target_body"] +===== `target_body` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"[body]" + ** ECS Compatibility enabled: no default value, needs to be specified explicitly + +Define the target field for placing the body of the HTTP response. + +[id="{version}-plugins-{type}s-{plugin}-target_headers"] +===== `target_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"[headers]"` + ** ECS Compatibility enabled: `"[@metadata][filter][http][response][headers]"` + +Define the target field for placing the headers of the HTTP response. + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value + +The URL to send the request to. The value can be fetched from event fields. + +[id="{version}-plugins-{type}s-{plugin}-verb"] +===== `verb` + + * Value type can be either `"GET"`, `"HEAD"`, `"PATCH"`, `"DELETE"`, `"POST"`, `"PUT"` + * Default value is `"GET"` + +The verb to be used for the HTTP request. + +[id="{version}-plugins-{type}s-{plugin}-connectivity-options"] +==== HTTP Filter Connectivity Options + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We highly recommend NOT setting this value +to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry! +Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="{version}-plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="{version}-plugins-{type}s-{plugin}-client_key"] +===== `client_key` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_key>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to be used in conjunction with the username for HTTP authentication. + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Timeout (in seconds) for the entire request. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried. + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL. +If you set this you must also set the `password` option. + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/http-v1.6.0.asciidoc b/docs/versioned-plugins/filters/http-v1.6.0.asciidoc new file mode 100644 index 000000000..2367849eb --- /dev/null +++ b/docs/versioned-plugins/filters/http-v1.6.0.asciidoc @@ -0,0 +1,541 @@ +:plugin: http +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.6.0 +:release_date: 2024-06-19 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-http/blob/v1.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== HTTP filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The HTTP filter provides integration with external web services/REST APIs. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +The plugin includes sensible defaults that change based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>. +When targeting an ECS version, headers are set as `@metadata` and the `target_body` is a required option. +See <<{version}-plugins-{type}s-{plugin}-target_body>>, and <<{version}-plugins-{type}s-{plugin}-target_headers>>. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== HTTP Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-body>> | String, Array or Hash |No +| <<{version}-plugins-{type}s-{plugin}-body_format>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-target_body>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target_headers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-verb>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +There are also multiple configuration options related to the HTTP connectivity: + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|no +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-body"] +===== `body` + + * Value type can be a {logstash-ref}/configuration-file-structure.html#string[string], {logstash-ref}/configuration-file-structure.html#number[number], {logstash-ref}/configuration-file-structure.html#array[array] or {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +The body of the HTTP request to be sent. + +An example to send `body` as json +``` +http { + body => { + "key1" => "constant_value" + "key2" => "%{[field][reference]}" + } + body_format => "json" +} +``` + +[id="{version}-plugins-{type}s-{plugin}-body_format"] +===== `body_format` + + * Value type can be either `"json"` or `"text"` + * Default value is `"text"` + +If set to `"json"` and the <<{version}-plugins-{type}s-{plugin}-body>> is a type of {logstash-ref}/configuration-file-structure.html#array[array] or {logstash-ref}/configuration-file-structure.html#hash[hash], the body will be serialized as JSON. Otherwise it is sent as is. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (for example, response headers target `headers` field by default) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, headers are added as metadata) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-target_body>> and +<<{version}-plugins-{type}s-{plugin}-target_headers>>. + + +[id="{version}-plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +The HTTP headers to be sent in the request. Both the names of the headers and their +values can reference values from event fields. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +Define the query string parameters (key-value pairs) to be sent in the HTTP request. + +[id="{version}-plugins-{type}s-{plugin}-target_body"] +===== `target_body` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"[body]" + ** ECS Compatibility enabled: no default value, needs to be specified explicitly + +Define the target field for placing the body of the HTTP response. + +[id="{version}-plugins-{type}s-{plugin}-target_headers"] +===== `target_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"[headers]"` + ** ECS Compatibility enabled: `"[@metadata][filter][http][response][headers]"` + +Define the target field for placing the headers of the HTTP response. + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value + +The URL to send the request to. The value can be fetched from event fields. + +[id="{version}-plugins-{type}s-{plugin}-verb"] +===== `verb` + + * Value type can be either `"GET"`, `"HEAD"`, `"PATCH"`, `"DELETE"`, `"POST"`, `"PUT"` + * Default value is `"GET"` + +The verb to be used for the HTTP request. + +[id="{version}-plugins-{type}s-{plugin}-connectivity-options"] +==== HTTP Filter Connectivity Options + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We highly recommend NOT setting this value +to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry! +Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="{version}-plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="{version}-plugins-{type}s-{plugin}-client_key"] +===== `client_key` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_key>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to be used in conjunction with the username for HTTP authentication. + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Timeout (in seconds) for the entire request. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried. + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` +deprecated[1.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL. +If you set this you must also set the `password` option. + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/http-v2.0.0.asciidoc b/docs/versioned-plugins/filters/http-v2.0.0.asciidoc new file mode 100644 index 000000000..4e02c9391 --- /dev/null +++ b/docs/versioned-plugins/filters/http-v2.0.0.asciidoc @@ -0,0 +1,474 @@ +:plugin: http +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v2.0.0 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-http/blob/v2.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== HTTP filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The HTTP filter provides integration with external web services/REST APIs. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +The plugin includes sensible defaults that change based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>. +When targeting an ECS version, headers are set as `@metadata` and the `target_body` is a required option. +See <<{version}-plugins-{type}s-{plugin}-target_body>>, and <<{version}-plugins-{type}s-{plugin}-target_headers>>. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== HTTP Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `2.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-body>> | String, Array or Hash |No +| <<{version}-plugins-{type}s-{plugin}-body_format>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-target_body>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target_headers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-verb>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +There are also multiple configuration options related to the HTTP connectivity: + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|no +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-body"] +===== `body` + + * Value type can be a {logstash-ref}/configuration-file-structure.html#string[string], {logstash-ref}/configuration-file-structure.html#number[number], {logstash-ref}/configuration-file-structure.html#array[array] or {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +The body of the HTTP request to be sent. + +An example to send `body` as json +``` +http { + body => { + "key1" => "constant_value" + "key2" => "%{[field][reference]}" + } + body_format => "json" +} +``` + +[id="{version}-plugins-{type}s-{plugin}-body_format"] +===== `body_format` + + * Value type can be either `"json"` or `"text"` + * Default value is `"text"` + +If set to `"json"` and the <<{version}-plugins-{type}s-{plugin}-body>> is a type of {logstash-ref}/configuration-file-structure.html#array[array] or {logstash-ref}/configuration-file-structure.html#hash[hash], the body will be serialized as JSON. Otherwise it is sent as is. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (for example, response headers target `headers` field by default) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, headers are added as metadata) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-target_body>> and +<<{version}-plugins-{type}s-{plugin}-target_headers>>. + + +[id="{version}-plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +The HTTP headers to be sent in the request. Both the names of the headers and their +values can reference values from event fields. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value + +Define the query string parameters (key-value pairs) to be sent in the HTTP request. + +[id="{version}-plugins-{type}s-{plugin}-target_body"] +===== `target_body` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"[body]" + ** ECS Compatibility enabled: no default value, needs to be specified explicitly + +Define the target field for placing the body of the HTTP response. + +[id="{version}-plugins-{type}s-{plugin}-target_headers"] +===== `target_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"[headers]"` + ** ECS Compatibility enabled: `"[@metadata][filter][http][response][headers]"` + +Define the target field for placing the headers of the HTTP response. + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value + +The URL to send the request to. The value can be fetched from event fields. + +[id="{version}-plugins-{type}s-{plugin}-verb"] +===== `verb` + + * Value type can be either `"GET"`, `"HEAD"`, `"PATCH"`, `"DELETE"`, `"POST"`, `"PUT"` + * Default value is `"GET"` + +The verb to be used for the HTTP request. + +[id="{version}-plugins-{type}s-{plugin}-connectivity-options"] +==== HTTP Filter Connectivity Options + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We highly recommend NOT setting this value +to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry! +Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to be used in conjunction with the username for HTTP authentication. + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Timeout (in seconds) for the entire request. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried. + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL. +If you set this you must also set the `password` option. + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Filter Obsolete Configuration Options + +WARNING: As of version `2.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| client_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| client_key |<<{version}-plugins-{type}s-{plugin}-ssl_key>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| keystore_type |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> +| truststore |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +| truststore_type |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-index.asciidoc b/docs/versioned-plugins/filters/jdbc_static-index.asciidoc index 4cbddd2fb..0465aaa2e 100644 --- a/docs/versioned-plugins/filters/jdbc_static-index.asciidoc +++ b/docs/versioned-plugins/filters/jdbc_static-index.asciidoc @@ -5,6 +5,16 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-05-30 +| <> | 2025-03-07 +| <> | 2024-12-23 +| <> | 2024-12-03 +| <> | 2024-08-09 +| <> | 2024-05-23 +| <> | 2024-05-08 +| <> | 2024-03-21 +| <> | 2024-02-13 +| <> | 2024-01-31 | <> | 2023-10-03 | <> | 2023-09-04 | <> | 2023-06-15 @@ -49,6 +59,16 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2018-01-11 |======================================================================= +include::jdbc_static-v5.6.0.asciidoc[] +include::jdbc_static-v5.5.3.asciidoc[] +include::jdbc_static-v5.5.2.asciidoc[] +include::jdbc_static-v5.5.1.asciidoc[] +include::jdbc_static-v5.5.0.asciidoc[] +include::jdbc_static-v5.4.11.asciidoc[] +include::jdbc_static-v5.4.10.asciidoc[] +include::jdbc_static-v5.4.9.asciidoc[] +include::jdbc_static-v5.4.8.asciidoc[] +include::jdbc_static-v5.4.7.asciidoc[] include::jdbc_static-v5.4.6.asciidoc[] include::jdbc_static-v5.4.5.asciidoc[] include::jdbc_static-v5.4.4.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.4.10.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.4.10.asciidoc new file mode 100644 index 000000000..aebef4f6e --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.4.10.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.10 +:release_date: 2024-05-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.10/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.4.11.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.4.11.asciidoc new file mode 100644 index 000000000..819da3170 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.4.11.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.11 +:release_date: 2024-05-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.11/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.4.7.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.4.7.asciidoc new file mode 100644 index 000000000..58cb4d20f --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.4.7.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.7 +:release_date: 2024-01-31 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.4.8.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.4.8.asciidoc new file mode 100644 index 000000000..d7e2d9d70 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.4.8.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.8 +:release_date: 2024-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.4.9.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.4.9.asciidoc new file mode 100644 index 000000000..8d108754a --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.4.9.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.9 +:release_date: 2024-03-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.5.0.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.5.0.asciidoc new file mode 100644 index 000000000..8ca86161c --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.5.0.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.0 +:release_date: 2024-08-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.5.1.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.5.1.asciidoc new file mode 100644 index 000000000..da941868e --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.5.1.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.1 +:release_date: 2024-12-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.5.2.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.5.2.asciidoc new file mode 100644 index 000000000..959cff64b --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.5.2.asciidoc @@ -0,0 +1,628 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.2 +:release_date: 2024-12-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" # run loaders every 2 hours + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.5.3.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.5.3.asciidoc new file mode 100644 index 000000000..ac76b2b8b --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.5.3.asciidoc @@ -0,0 +1,629 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.3 +:release_date: 2025-03-07 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" <8> + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. +<8> Runs loaders every 2 hours. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.6.0.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.6.0.asciidoc new file mode 100644 index 000000000..eacdcd047 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.6.0.asciidoc @@ -0,0 +1,629 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.0 +:release_date: 2025-05-30 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" <8> + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. +<8> Runs loaders every 2 hours. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc index ccde2f51e..a15128b26 100644 --- a/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc +++ b/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc @@ -5,6 +5,16 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-05-30 +| <> | 2025-03-07 +| <> | 2024-12-23 +| <> | 2024-12-03 +| <> | 2024-08-09 +| <> | 2024-05-23 +| <> | 2024-05-08 +| <> | 2024-03-21 +| <> | 2024-02-13 +| <> | 2024-01-31 | <> | 2023-10-03 | <> | 2023-09-04 | <> | 2023-06-15 @@ -49,6 +59,16 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::jdbc_streaming-v5.6.0.asciidoc[] +include::jdbc_streaming-v5.5.3.asciidoc[] +include::jdbc_streaming-v5.5.2.asciidoc[] +include::jdbc_streaming-v5.5.1.asciidoc[] +include::jdbc_streaming-v5.5.0.asciidoc[] +include::jdbc_streaming-v5.4.11.asciidoc[] +include::jdbc_streaming-v5.4.10.asciidoc[] +include::jdbc_streaming-v5.4.9.asciidoc[] +include::jdbc_streaming-v5.4.8.asciidoc[] +include::jdbc_streaming-v5.4.7.asciidoc[] include::jdbc_streaming-v5.4.6.asciidoc[] include::jdbc_streaming-v5.4.5.asciidoc[] include::jdbc_streaming-v5.4.4.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.4.10.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.10.asciidoc new file mode 100644 index 000000000..dad282031 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.10.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.10 +:release_date: 2024-05-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.10/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.4.11.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.11.asciidoc new file mode 100644 index 000000000..98e72356d --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.11.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.11 +:release_date: 2024-05-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.11/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.4.7.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.7.asciidoc new file mode 100644 index 000000000..88479e569 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.7.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.7 +:release_date: 2024-01-31 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.4.8.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.8.asciidoc new file mode 100644 index 000000000..07a5138a1 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.8.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.8 +:release_date: 2024-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.4.9.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.9.asciidoc new file mode 100644 index 000000000..37faac1b0 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.4.9.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.9 +:release_date: 2024-03-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.5.0.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.0.asciidoc new file mode 100644 index 000000000..9e1dfba27 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.0.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.0 +:release_date: 2024-08-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.5.1.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.1.asciidoc new file mode 100644 index 000000000..20ea4c4aa --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.1.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.1 +:release_date: 2024-12-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.5.2.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.2.asciidoc new file mode 100644 index 000000000..9250c2793 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.2.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.2 +:release_date: 2024-12-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.5.3.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.3.asciidoc new file mode 100644 index 000000000..4d4656682 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.5.3.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.3 +:release_date: 2025-03-07 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.6.0.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.6.0.asciidoc new file mode 100644 index 000000000..873cb3b72 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.6.0.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.0 +:release_date: 2025-05-30 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/json-index.asciidoc b/docs/versioned-plugins/filters/json-index.asciidoc index 38d912345..f77d70c12 100644 --- a/docs/versioned-plugins/filters/json-index.asciidoc +++ b/docs/versioned-plugins/filters/json-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2023-12-18 | <> | 2021-06-17 | <> | 2019-06-17 | <> | 2019-02-04 @@ -13,6 +14,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::json-v3.2.1.asciidoc[] include::json-v3.2.0.asciidoc[] include::json-v3.1.0.asciidoc[] include::json-v3.0.6.asciidoc[] diff --git a/docs/versioned-plugins/filters/json-v3.2.1.asciidoc b/docs/versioned-plugins/filters/json-v3.2.1.asciidoc new file mode 100644 index 000000000..dbe9b4216 --- /dev/null +++ b/docs/versioned-plugins/filters/json-v3.2.1.asciidoc @@ -0,0 +1,141 @@ +:plugin: json +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.2.1 +:release_date: 2023-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-json/blob/v3.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JSON filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This is a JSON parsing filter. It takes an existing field which contains JSON and +expands it into an actual data structure within the Logstash event. + +By default, it will place the parsed JSON in the root (top level) of the Logstash event, but this +filter can be configured to place the JSON into any arbitrary event field, using the +`target` configuration. + +This plugin has a few fallback scenarios when something bad happens during the parsing of the event. +If the JSON parsing fails on the data, the event will be untouched and it will be tagged with +`_jsonparsefailure`; you can then use conditionals to clean the data. You can configure this tag with the +`tag_on_failure` option. + +If the parsed data contains a `@timestamp` field, the plugin will try to use it for the events `@timestamp`, and if the +parsing fails, the field will be renamed to `_@timestamp` and the event will be tagged with a +`_timestampparsefailure`. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +The plugin behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== JSON Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-skip_on_invalid_json>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not use ECS-compatible field names +** `v1`: Elastic Common Schema compliant behavior (warns when `target` isn't set) + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-skip_on_invalid_json"] +===== `skip_on_invalid_json` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Allows for skipping the filter on invalid JSON (this allows you to handle JSON and non-JSON data without warnings) + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The configuration for the JSON filter: +[source,ruby] + source => source_field + +For example, if you have JSON data in the `message` field: +[source,ruby] + filter { + json { + source => "message" + } + } + +The above would parse the JSON from the `message` field. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jsonparsefailure"]` + +Append values to the `tags` field when there has been no +successful match + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the parsed data. If this setting is +omitted, the JSON data will be stored at the root (top level) of the event. + +For example, if you want the data to be put in the `doc` field: +[source,ruby] + filter { + json { + target => "doc" + } + } + +JSON in the value of the `source` field will be expanded into a +data structure in the `target` field. + +NOTE: if the `target` field already exists, it will be overwritten! + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/syslog_pri-index.asciidoc b/docs/versioned-plugins/filters/syslog_pri-index.asciidoc index ec7e1112f..1c665a873 100644 --- a/docs/versioned-plugins/filters/syslog_pri-index.asciidoc +++ b/docs/versioned-plugins/filters/syslog_pri-index.asciidoc @@ -5,6 +5,8 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-01-17 +| <> | 2023-01-11 | <> | 2021-11-10 | <> | 2021-03-16 | <> | 2017-11-07 @@ -12,6 +14,8 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::syslog_pri-v3.2.1.asciidoc[] +include::syslog_pri-v3.2.0.asciidoc[] include::syslog_pri-v3.1.1.asciidoc[] include::syslog_pri-v3.1.0.asciidoc[] include::syslog_pri-v3.0.5.asciidoc[] diff --git a/docs/versioned-plugins/filters/syslog_pri-v3.2.0.asciidoc b/docs/versioned-plugins/filters/syslog_pri-v3.2.0.asciidoc new file mode 100644 index 000000000..88210a741 --- /dev/null +++ b/docs/versioned-plugins/filters/syslog_pri-v3.2.0.asciidoc @@ -0,0 +1,104 @@ +:plugin: syslog_pri +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.2.0 +:release_date: 2023-01-11 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-syslog_pri/blob/v3.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Syslog_pri filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Filter plugin for logstash to parse the `PRI` field from the front +of a Syslog (RFC3164) message. If no priority is set, it will +default to 13 (per RFC). + +This filter is based on the original `syslog.rb` code shipped +with logstash. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Syslog_pri Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-facility_labels>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-severity_labels>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-syslog_pri_field_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_labels>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not use ECS-compatible field names (for example, `syslog_severity_code` for syslog severity) +** `v1`, `v8`: uses fields that are compatible with Elastic Common Schema (for example, `[log][syslog][severity][code]`) +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-syslog_pri_field_name>>. + +[id="{version}-plugins-{type}s-{plugin}-facility_labels"] +===== `facility_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["kernel", "user-level", "mail", "daemon", "security/authorization", "syslogd", "line printer", "network news", "uucp", "clock", "security/authorization", "ftp", "ntp", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7"]` + +Labels for facility levels. This comes from RFC3164. +If an unrecognized facility code is provided and <<{version}-plugins-{type}s-{plugin}-use_labels>> is `true` then the event +is tagged with `_syslogpriparsefailure`. + +[id="{version}-plugins-{type}s-{plugin}-severity_labels"] +===== `severity_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["emergency", "alert", "critical", "error", "warning", "notice", "informational", "debug"]` + +Labels for severity levels. This comes from RFC3164. + +[id="{version}-plugins-{type}s-{plugin}-syslog_pri_field_name"] +===== `syslog_pri_field_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"syslog_pri"` + ** ECS Compatibility enabled: `"[log][syslog][priority]"` + +Name of field which passes in the extracted PRI part of the syslog message + +[id="{version}-plugins-{type}s-{plugin}-use_labels"] +===== `use_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +set the status to experimental/beta/stable +Add human-readable names after parsing severity and facility from PRI + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] \ No newline at end of file diff --git a/docs/versioned-plugins/filters/syslog_pri-v3.2.1.asciidoc b/docs/versioned-plugins/filters/syslog_pri-v3.2.1.asciidoc new file mode 100644 index 000000000..52b57a27c --- /dev/null +++ b/docs/versioned-plugins/filters/syslog_pri-v3.2.1.asciidoc @@ -0,0 +1,103 @@ +:plugin: syslog_pri +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.2.1 +:release_date: 2024-01-17 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-syslog_pri/blob/v3.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Syslog_pri filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Filter plugin for logstash to parse the `PRI` field from the front +of a Syslog (RFC3164) message. If no priority is set, it will +default to 13 (per RFC). + +This filter is based on the original `syslog.rb` code shipped +with logstash. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Syslog_pri Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-facility_labels>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-severity_labels>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-syslog_pri_field_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_labels>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not use ECS-compatible field names (for example, `syslog_severity_code` for syslog severity) +** `v1`, `v8`: uses fields that are compatible with Elastic Common Schema (for example, `[log][syslog][severity][code]`) +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-syslog_pri_field_name>>. + +[id="{version}-plugins-{type}s-{plugin}-facility_labels"] +===== `facility_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["kernel", "user-level", "mail", "daemon", "security/authorization", "syslogd", "line printer", "network news", "uucp", "clock", "security/authorization", "ftp", "ntp", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7"]` + +Labels for facility levels. This comes from RFC3164. +If an unrecognized facility code is provided and <<{version}-plugins-{type}s-{plugin}-use_labels>> is `true` then the event +is tagged with `_syslogpriparsefailure`. + +[id="{version}-plugins-{type}s-{plugin}-severity_labels"] +===== `severity_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["emergency", "alert", "critical", "error", "warning", "notice", "informational", "debug"]` + +Labels for severity levels. This comes from RFC3164. + +[id="{version}-plugins-{type}s-{plugin}-syslog_pri_field_name"] +===== `syslog_pri_field_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"syslog_pri"` + ** ECS Compatibility enabled: `"[log][syslog][priority]"` + +Name of field which passes in the extracted PRI part of the syslog message + +[id="{version}-plugins-{type}s-{plugin}-use_labels"] +===== `use_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Add human-readable names after parsing severity and facility from PRI + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/translate-index.asciidoc b/docs/versioned-plugins/filters/translate-index.asciidoc index 25e690e52..c3c8f8fe8 100644 --- a/docs/versioned-plugins/filters/translate-index.asciidoc +++ b/docs/versioned-plugins/filters/translate-index.asciidoc @@ -5,6 +5,8 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-08-04 +| <> | 2025-07-24 | <> | 2023-06-14 | <> | 2023-05-12 | <> | 2022-06-08 @@ -20,6 +22,8 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::translate-v3.5.0.asciidoc[] +include::translate-v3.4.3.asciidoc[] include::translate-v3.4.2.asciidoc[] include::translate-v3.4.1.asciidoc[] include::translate-v3.4.0.asciidoc[] diff --git a/docs/versioned-plugins/filters/translate-v3.4.3.asciidoc b/docs/versioned-plugins/filters/translate-v3.4.3.asciidoc new file mode 100644 index 000000000..f9c88df96 --- /dev/null +++ b/docs/versioned-plugins/filters/translate-v3.4.3.asciidoc @@ -0,0 +1,436 @@ +:plugin: translate +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.4.3 +:release_date: 2025-07-24 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-translate/blob/v3.4.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Translate filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +A general search and replace tool that uses a configured hash +and/or a file to determine replacement values. Currently supported are +YAML, JSON, and CSV files. Each dictionary item is a key value pair. + +You can specify dictionary entries in one of two ways: + +* The `dictionary` configuration item can contain a hash representing +the mapping. +* An external file (readable by logstash) may be specified in the +`dictionary_path` configuration item. + +These two methods may not be used in conjunction; it will produce an error. + +Operationally, for each event, the value from the `source` setting is tested +against the dictionary and if it matches exactly (or matches a regex when +`regex` configuration item has been enabled), the matched value is put in the +`target` field, but on no match the `fallback` setting string is used instead. + +Example: +[source,ruby] +---- + filter { + translate { + source => "[http][response][status_code]" + target => "[http_status_description]" + dictionary => { + "100" => "Continue" + "101" => "Switching Protocols" + "200" => "OK" + "500" => "Server Error" + } + fallback => "I'm a teapot" + } + } +---- + +Occasionally, people find that they have a field with a variable sized array of +values or objects that need some enrichment. The `iterate_on` setting helps in +these cases. + +Alternatively, for simple string search and replacements for just a few values +you might consider using the gsub function of the mutate filter. + +It is possible to provide multi-valued dictionary values. When using a YAML or +JSON dictionary, you can have the value as a hash (map) or an array datatype. +When using a CSV dictionary, multiple values in the translation must be +extracted with another filter e.g. Dissect or KV. + +Note that the `fallback` is a string so on no match the fallback setting needs +to formatted so that a filter can extract the multiple values to the correct fields. + +File based dictionaries are loaded in a separate thread using a scheduler. +If you set a `refresh_interval` of 300 seconds (5 minutes) or less then the +modified time of the file is checked before reloading. Very large dictionaries +are supported, internally tested at 100 000 key/values, and we minimise +the impact on throughput by having the refresh in the scheduler thread. +Any ongoing modification of the dictionary file should be done using a +copy/edit/rename or create/rename mechanism to avoid the refresh code from +processing half-baked dictionary content. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Compatibility with the Elastic Common Schema (ECS) + +The plugin acts as an in-place translator if `source` and `target` are the same +and does not produce any new event fields. +This is the default behavior in <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Translate Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-destination>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dictionary>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-dictionary_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exact>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fallback>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-iterate_on>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-override>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-refresh_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-regex>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-refresh_behaviour>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-yaml_dictionary_code_point_limit>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-destination"] +===== `destination` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Deprecated alias for <<{version}-plugins-{type}s-{plugin}-target>> setting. + +deprecated[3.3.0, Use <<{version}-plugins-{type}s-{plugin}-target>> instead. In 4.0 this setting will be removed.] + +[id="{version}-plugins-{type}s-{plugin}-dictionary"] +===== `dictionary` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The dictionary to use for translation, when specified in the logstash filter +configuration item (i.e. do not use the `dictionary_path` file). + +Example: +[source,ruby] +---- + filter { + translate { + dictionary => { + "100" => "Continue" + "101" => "Switching Protocols" + "merci" => "thank you" + "old version" => "new version" + } + } + } +---- + +NOTE: It is an error to specify both `dictionary` and `dictionary_path`. + +[id="{version}-plugins-{type}s-{plugin}-dictionary_path"] +===== `dictionary_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The full path of the external dictionary file. The format of the table should be +a standard YAML, JSON, or CSV. + +Specify any integer-based keys in quotes. The value taken from the event's +`source` setting is converted to a string. The lookup dictionary keys must also +be strings, and the quotes make the integer-based keys function as a string. +For example, the YAML file should look something like this: + +[source,ruby] +---- + "100": Continue + "101": Switching Protocols + merci: gracias + old version: new version +---- + +NOTE: It is an error to specify both `dictionary` and `dictionary_path`. + +The currently supported formats are YAML, JSON, and CSV. Format selection is +based on the file extension: `json` for JSON, `yaml` or `yml` for YAML, and +`csv` for CSV. The CSV format expects exactly two columns, with the first serving +as the original text (lookup key), and the second column as the translation. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: disabled ECS-compatibility + ** `v1`, `v8`: compatibility with the specified major version of the Elastic Common Schema + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-target>>. + +[id="{version}-plugins-{type}s-{plugin}-exact"] +===== `exact` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +When `exact => true`, the translate filter will populate the destination field +with the exact contents of the dictionary value. When `exact => false`, the +filter will populate the destination field with the result of any existing +destination field's data, with the translated value substituted in-place. + +For example, consider this simple translation.yml, configured to check the `data` field: +[source,ruby] +---- + foo: bar +---- + +If logstash receives an event with the `data` field set to `foo`, and `exact => true`, +the destination field will be populated with the string `bar`. +If `exact => false`, and logstash receives the same event, the destination field +will be also set to `bar`. However, if logstash receives an event with the `data` field +set to `foofing`, the destination field will be set to `barfing`. + +Set both `exact => true` AND `regex => `true` if you would like to match using dictionary +keys as regular expressions. A large dictionary could be expensive to match in this case. + +[id="{version}-plugins-{type}s-{plugin}-fallback"] +===== `fallback` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +In case no translation occurs in the event (no matches), this will add a default +translation string, which will always populate `field`, if the match failed. + +For example, if we have configured `fallback => "no match"`, using this dictionary: +[source,ruby] +---- + foo: bar +---- + +Then, if logstash received an event with the field `foo` set to `bar`, the destination +field would be set to `bar`. However, if logstash received an event with `foo` set to `nope`, +then the destination field would still be populated, but with the value of `no match`. +This configuration can be dynamic and include parts of the event using the `%{field}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-field"] +===== `field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Deprecated alias for <<{version}-plugins-{type}s-{plugin}-source>> setting. + +deprecated[3.3.0, Use <<{version}-plugins-{type}s-{plugin}-source>> instead. In 4.0 this setting will be removed.] + +[id="{version}-plugins-{type}s-{plugin}-iterate_on"] +===== `iterate_on` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When the value that you need to perform enrichment on is a variable sized array +then specify the field name in this setting. This setting introduces two modes, +1) when the value is an array of strings and 2) when the value is an array of +objects (as in JSON object). + +In the first mode, you should have the same field name in both `source` and +`iterate_on`, the result will be an array added to the field specified in the +`target` setting. This array will have the looked up value (or the +`fallback` value or nil) in same ordinal position as each sought value. + +In the second mode, specify the field that has the array of objects in +`iterate_on` then specify the field in each object that provides the sought value +with `source` and the field to write the looked up value (or the `fallback` value) +to with `target`. + +For a dictionary of: +[source,csv] +---- + 100,Yuki + 101,Rupert + 102,Ahmed + 103,Kwame +---- + +Example of Mode 1 +[source,ruby] + filter { + translate { + iterate_on => "[collaborator_ids]" + source => "[collaborator_ids]" + target => "[collaborator_names]" + fallback => "Unknown" + } + } + +Before +[source,json] + { + "collaborator_ids": [100,103,110,102] + } + +After +[source,json] + { + "collaborator_ids": [100,103,110,102], + "collabrator_names": ["Yuki","Kwame","Unknown","Ahmed"] + } + +Example of Mode 2 +[source,ruby] + filter { + translate { + iterate_on => "[collaborators]" + source => "[id]" + target => "[name]" + fallback => "Unknown" + } + } + +Before +[source,json] + { + "collaborators": [ + { + "id": 100 + }, + { + "id": 103 + }, + { + "id": 110 + }, + { + "id": 101 + } + ] + } + +After +[source,json] + { + "collaborators": [ + { + "id": 100, + "name": "Yuki" + }, + { + "id": 103, + "name": "Kwame" + }, + { + "id": 110, + "name": "Unknown" + }, + { + "id": 101, + "name": "Rupert" + } + ] + } + +[id="{version}-plugins-{type}s-{plugin}-override"] +===== `override` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value depends on whether in-place translation is being used + +If the destination (or target) field already exists, this configuration option controls +whether the filter skips translation (default behavior) or overwrites the target +field value with the new translation value. + +In case of in-place translation, where `target` is the same as `source` (such as when +<<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled), overwriting is allowed. + +[id="{version}-plugins-{type}s-{plugin}-refresh_interval"] +===== `refresh_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300` + +When using a dictionary file, this setting will indicate how frequently +(in seconds) logstash will check the dictionary file for updates. + +A value of zero or less will disable refresh. + +[id="{version}-plugins-{type}s-{plugin}-regex"] +===== `regex` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +To treat dictionary keys as regular expressions, set `regex => true`. + +Be sure to escape dictionary key strings for use with regex. +Resources on regex formatting are available online. + +[id="{version}-plugins-{type}s-{plugin}-refresh_behaviour"] +===== `refresh_behaviour` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `merge` + +When using a dictionary file, this setting indicates how the update will be executed. +Setting this to `merge` causes the new dictionary to be merged into the old one. This means +same entry will be updated but entries that existed before but not in the new dictionary +will remain after the merge; `replace` causes the whole dictionary to be replaced +with a new one (deleting all entries of the old one on update). + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + +* This is a required setting. +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The name of the logstash event field containing the value to be compared for a +match by the translate filter (e.g. `message`, `host`, `response_code`). + +If this field is an array, only the first value will be used. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"translation"` + ** ECS Compatibility enabled: defaults to the same value as `source` + +The target field you wish to populate with the translated code. +If you set this value to the same value as `source` field, the plugin does a substitution, and +the filter will succeed. This will clobber the old value of the source field! + + +[id="{version}-plugins-{type}s-{plugin}-yaml_dictionary_code_point_limit"] +===== `yaml_dictionary_code_point_limit` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is 134217728 (128MB for 1 byte code points) + +The max amount of code points in the YAML file in `dictionary_path`. Please be aware that byte limit depends on the encoding. +This setting is effective for YAML file only. YAML over the limit throws exception. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/translate-v3.5.0.asciidoc b/docs/versioned-plugins/filters/translate-v3.5.0.asciidoc new file mode 100644 index 000000000..97060bb08 --- /dev/null +++ b/docs/versioned-plugins/filters/translate-v3.5.0.asciidoc @@ -0,0 +1,453 @@ +:plugin: translate +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.5.0 +:release_date: 2025-08-04 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-translate/blob/v3.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Translate filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +A general search and replace tool that uses a configured hash +and/or a file to determine replacement values. Currently supported are +YAML, JSON, and CSV files. Each dictionary item is a key value pair. + +You can specify dictionary entries in one of two ways: + +* The `dictionary` configuration item can contain a hash representing +the mapping. +* An external file (readable by logstash) may be specified in the +`dictionary_path` configuration item. + +These two methods may not be used in conjunction; it will produce an error. + +Operationally, for each event, the value from the `source` setting is tested +against the dictionary and if it matches exactly (or matches a regex when +`regex` configuration item has been enabled), the matched value is put in the +`target` field, but on no match the `fallback` setting string is used instead. + +Example: +[source,ruby] +---- + filter { + translate { + source => "[http][response][status_code]" + target => "[http_status_description]" + dictionary => { + "100" => "Continue" + "101" => "Switching Protocols" + "200" => "OK" + "500" => "Server Error" + } + fallback => "I'm a teapot" + } + } +---- + +Occasionally, people find that they have a field with a variable sized array of +values or objects that need some enrichment. The `iterate_on` setting helps in +these cases. + +Alternatively, for simple string search and replacements for just a few values +you might consider using the gsub function of the mutate filter. + +It is possible to provide multi-valued dictionary values. When using a YAML or +JSON dictionary, you can have the value as a hash (map) or an array datatype. +When using a CSV dictionary, multiple values in the translation must be +extracted with another filter e.g. Dissect or KV. + +Note that the `fallback` is a string so on no match the fallback setting needs +to formatted so that a filter can extract the multiple values to the correct fields. + +File based dictionaries are loaded in a separate thread using a scheduler. +If you set a `refresh_interval` of 300 seconds (5 minutes) or less then the +modified time of the file is checked before reloading. Very large dictionaries +are supported, internally tested at 100 000 key/values, and we minimise +the impact on throughput by having the refresh in the scheduler thread. +Any ongoing modification of the dictionary file should be done using a +copy/edit/rename or create/rename mechanism to avoid the refresh code from +processing half-baked dictionary content. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Compatibility with the Elastic Common Schema (ECS) + +The plugin acts as an in-place translator if `source` and `target` are the same +and does not produce any new event fields. +This is the default behavior in <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Translate Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-destination>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dictionary>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-dictionary_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exact>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fallback>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-iterate_on>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-override>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-refresh_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-regex>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-refresh_behaviour>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-yaml_dictionary_code_point_limit>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-yaml_load_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["one_shot", "streaming"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-destination"] +===== `destination` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Deprecated alias for <<{version}-plugins-{type}s-{plugin}-target>> setting. + +deprecated[3.3.0, Use <<{version}-plugins-{type}s-{plugin}-target>> instead. In 4.0 this setting will be removed.] + +[id="{version}-plugins-{type}s-{plugin}-dictionary"] +===== `dictionary` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The dictionary to use for translation, when specified in the logstash filter +configuration item (i.e. do not use the `dictionary_path` file). + +Example: +[source,ruby] +---- + filter { + translate { + dictionary => { + "100" => "Continue" + "101" => "Switching Protocols" + "merci" => "thank you" + "old version" => "new version" + } + } + } +---- + +NOTE: It is an error to specify both `dictionary` and `dictionary_path`. + +[id="{version}-plugins-{type}s-{plugin}-dictionary_path"] +===== `dictionary_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The full path of the external dictionary file. The format of the table should be +a standard YAML, JSON, or CSV. + +Specify any integer-based keys in quotes. The value taken from the event's +`source` setting is converted to a string. The lookup dictionary keys must also +be strings, and the quotes make the integer-based keys function as a string. +For example, the YAML file should look something like this: + +[source,ruby] +---- + "100": Continue + "101": Switching Protocols + merci: gracias + old version: new version +---- + +NOTE: It is an error to specify both `dictionary` and `dictionary_path`. + +The currently supported formats are YAML, JSON, and CSV. Format selection is +based on the file extension: `json` for JSON, `yaml` or `yml` for YAML, and +`csv` for CSV. The CSV format expects exactly two columns, with the first serving +as the original text (lookup key), and the second column as the translation. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: disabled ECS-compatibility + ** `v1`, `v8`: compatibility with the specified major version of the Elastic Common Schema + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-target>>. + +[id="{version}-plugins-{type}s-{plugin}-exact"] +===== `exact` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +When `exact => true`, the translate filter will populate the destination field +with the exact contents of the dictionary value. When `exact => false`, the +filter will populate the destination field with the result of any existing +destination field's data, with the translated value substituted in-place. + +For example, consider this simple translation.yml, configured to check the `data` field: +[source,ruby] +---- + foo: bar +---- + +If logstash receives an event with the `data` field set to `foo`, and `exact => true`, +the destination field will be populated with the string `bar`. +If `exact => false`, and logstash receives the same event, the destination field +will be also set to `bar`. However, if logstash receives an event with the `data` field +set to `foofing`, the destination field will be set to `barfing`. + +Set both `exact => true` AND `regex => `true` if you would like to match using dictionary +keys as regular expressions. A large dictionary could be expensive to match in this case. + +[id="{version}-plugins-{type}s-{plugin}-fallback"] +===== `fallback` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +In case no translation occurs in the event (no matches), this will add a default +translation string, which will always populate `field`, if the match failed. + +For example, if we have configured `fallback => "no match"`, using this dictionary: +[source,ruby] +---- + foo: bar +---- + +Then, if logstash received an event with the field `foo` set to `bar`, the destination +field would be set to `bar`. However, if logstash received an event with `foo` set to `nope`, +then the destination field would still be populated, but with the value of `no match`. +This configuration can be dynamic and include parts of the event using the `%{field}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-field"] +===== `field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Deprecated alias for <<{version}-plugins-{type}s-{plugin}-source>> setting. + +deprecated[3.3.0, Use <<{version}-plugins-{type}s-{plugin}-source>> instead. In 4.0 this setting will be removed.] + +[id="{version}-plugins-{type}s-{plugin}-iterate_on"] +===== `iterate_on` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When the value that you need to perform enrichment on is a variable sized array +then specify the field name in this setting. This setting introduces two modes, +1) when the value is an array of strings and 2) when the value is an array of +objects (as in JSON object). + +In the first mode, you should have the same field name in both `source` and +`iterate_on`, the result will be an array added to the field specified in the +`target` setting. This array will have the looked up value (or the +`fallback` value or nil) in same ordinal position as each sought value. + +In the second mode, specify the field that has the array of objects in +`iterate_on` then specify the field in each object that provides the sought value +with `source` and the field to write the looked up value (or the `fallback` value) +to with `target`. + +For a dictionary of: +[source,csv] +---- + 100,Yuki + 101,Rupert + 102,Ahmed + 103,Kwame +---- + +Example of Mode 1 +[source,ruby] + filter { + translate { + iterate_on => "[collaborator_ids]" + source => "[collaborator_ids]" + target => "[collaborator_names]" + fallback => "Unknown" + } + } + +Before +[source,json] + { + "collaborator_ids": [100,103,110,102] + } + +After +[source,json] + { + "collaborator_ids": [100,103,110,102], + "collabrator_names": ["Yuki","Kwame","Unknown","Ahmed"] + } + +Example of Mode 2 +[source,ruby] + filter { + translate { + iterate_on => "[collaborators]" + source => "[id]" + target => "[name]" + fallback => "Unknown" + } + } + +Before +[source,json] + { + "collaborators": [ + { + "id": 100 + }, + { + "id": 103 + }, + { + "id": 110 + }, + { + "id": 101 + } + ] + } + +After +[source,json] + { + "collaborators": [ + { + "id": 100, + "name": "Yuki" + }, + { + "id": 103, + "name": "Kwame" + }, + { + "id": 110, + "name": "Unknown" + }, + { + "id": 101, + "name": "Rupert" + } + ] + } + +[id="{version}-plugins-{type}s-{plugin}-override"] +===== `override` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value depends on whether in-place translation is being used + +If the destination (or target) field already exists, this configuration option controls +whether the filter skips translation (default behavior) or overwrites the target +field value with the new translation value. + +In case of in-place translation, where `target` is the same as `source` (such as when +<<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled), overwriting is allowed. + +[id="{version}-plugins-{type}s-{plugin}-refresh_interval"] +===== `refresh_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300` + +When using a dictionary file, this setting will indicate how frequently +(in seconds) logstash will check the dictionary file for updates. + +A value of zero or less will disable refresh. + +[id="{version}-plugins-{type}s-{plugin}-regex"] +===== `regex` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +To treat dictionary keys as regular expressions, set `regex => true`. + +Be sure to escape dictionary key strings for use with regex. +Resources on regex formatting are available online. + +[id="{version}-plugins-{type}s-{plugin}-refresh_behaviour"] +===== `refresh_behaviour` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `merge` + +When using a dictionary file, this setting indicates how the update will be executed. +Setting this to `merge` causes the new dictionary to be merged into the old one. This means +same entry will be updated but entries that existed before but not in the new dictionary +will remain after the merge; `replace` causes the whole dictionary to be replaced +with a new one (deleting all entries of the old one on update). + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + +* This is a required setting. +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The name of the logstash event field containing the value to be compared for a +match by the translate filter (e.g. `message`, `host`, `response_code`). + +If this field is an array, only the first value will be used. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"translation"` + ** ECS Compatibility enabled: defaults to the same value as `source` + +The target field you wish to populate with the translated code. +If you set this value to the same value as `source` field, the plugin does a substitution, and +the filter will succeed. This will clobber the old value of the source field! + + +[id="{version}-plugins-{type}s-{plugin}-yaml_dictionary_code_point_limit"] +===== `yaml_dictionary_code_point_limit` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is 134217728 (128MB for 1 byte code points) + +The max amount of code points in the YAML file in `dictionary_path`. Please be aware that byte limit depends on the encoding. +This setting is effective for YAML file only. YAML over the limit throws exception. + +[id="{version}-plugins-{type}s-{plugin}-yaml_load_strategy"] +===== `yaml_load_strategy` + +* Value can be any of: `one_shot`, `streaming` +* Default value is `one_shot` + +How to load and parse the YAML file. This setting defaults to `one_shot`, which loads the entire +YAML file into the parser in one go, emitting the final dictionary from the fully parsed YAML document. + +Setting to `streaming` will instead instruct the parser to emit one "YAML element" at a time, constructing the dictionary +during parsing. This mode drastically reduces the amount of memory required to load or refresh the dictionary and it is also faster. + +Due to underlying implementation differences this mode only supports basic types such as Arrays, Objects, Strings, numbers and booleans, and does not support tags. + +If you have a lot of translate filters with large YAML documents consider changing this setting to `streaming` instead. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/xml-index.asciidoc b/docs/versioned-plugins/filters/xml-index.asciidoc index b16db3cd7..7eeb61b12 100644 --- a/docs/versioned-plugins/filters/xml-index.asciidoc +++ b/docs/versioned-plugins/filters/xml-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-07-24 +| <> | 2025-04-22 +| <> | 2025-02-19 +| <> | 2024-10-29 | <> | 2022-08-17 | <> | 2021-12-07 | <> | 2021-06-23 @@ -17,6 +21,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::xml-v4.3.2.asciidoc[] +include::xml-v4.3.1.asciidoc[] +include::xml-v4.3.0.asciidoc[] +include::xml-v4.2.1.asciidoc[] include::xml-v4.2.0.asciidoc[] include::xml-v4.1.3.asciidoc[] include::xml-v4.1.2.asciidoc[] diff --git a/docs/versioned-plugins/filters/xml-v4.2.1.asciidoc b/docs/versioned-plugins/filters/xml-v4.2.1.asciidoc new file mode 100644 index 000000000..f996346ee --- /dev/null +++ b/docs/versioned-plugins/filters/xml-v4.2.1.asciidoc @@ -0,0 +1,213 @@ +:plugin: xml +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.2.1 +:release_date: 2024-10-29 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-xml/blob/v4.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Xml filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +XML filter. Takes a field that contains XML and expands it into +an actual datastructure. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Xml Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-force_array>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-force_content>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-namespaces>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parse_options>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-remove_namespaces>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-store_xml>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-suppress_empty>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-xpath>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-force_array"] +===== `force_array` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will force single elements to be arrays. Setting this to +false will prevent storing single elements in arrays. + +[id="{version}-plugins-{type}s-{plugin}-force_content"] +===== `force_content` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +By default the filter will expand attributes differently from content inside +of tags. This option allows you to force text content and attributes to always +parse to a hash value. + +[id="{version}-plugins-{type}s-{plugin}-namespaces"] +===== `namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +By default only namespaces declarations on the root element are considered. +This allows to configure all namespace declarations to parse the XML document. + +Example: + +[source,ruby] +filter { + xml { + namespaces => { + "xsl" => "http://www.w3.org/1999/XSL/Transform" + "xhtml" => "http://www.w3.org/1999/xhtml" + } + } +} + +[id="{version}-plugins-{type}s-{plugin}-parse_options"] +===== `parse_options` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Setting XML parse options allows for more control of the parsing process. +By default the parser is not strict and thus accepts some invalid content. +Currently supported options are: + + - `strict` - forces the parser to fail early instead of accumulating errors when content is not valid xml. + +Control characters such as ASCII 0x0 are not allowed and _always_ result in non-valid XML. + +When XML content is not valid, it will be tagged as `_xmlparsefailure`. + +XML specs: + +* XML 1.0 Spec: https://www.w3.org/TR/2008/REC-xml-20081126/#charsets +* XML 1.1 Spec: https://www.w3.org/TR/xml11/#charsets + + + + +[id="{version}-plugins-{type}s-{plugin}-remove_namespaces"] +===== `remove_namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Remove all namespaces from all nodes in the document. +Of course, if the document had nodes with the same names but different namespaces, they will now be ambiguous. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Config for xml to hash is: +[source,ruby] + source => source_field + +For example, if you have the whole XML document in your `message` field: +[source,ruby] + filter { + xml { + source => "message" + } + } + +The above would parse the XML from the `message` field. + +[id="{version}-plugins-{type}s-{plugin}-store_xml"] +===== `store_xml` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will store the whole parsed XML in the destination +field as described above. Setting this to false will prevent that. + +[id="{version}-plugins-{type}s-{plugin}-suppress_empty"] +===== `suppress_empty` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default, output nothing if the element is empty. +If set to `false`, empty element will result in an empty hash object. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define target for placing the data + +For example if you want the data to be put in the `doc` field: +[source,ruby] + filter { + xml { + target => "doc" + } + } + +XML in the value of the source field will be expanded into a +datastructure in the `target` field. +Note: if the `target` field already exists, it will be overridden. +Required if `store_xml` is true (which is the default). + +[id="{version}-plugins-{type}s-{plugin}-xpath"] +===== `xpath` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +xpath will additionally select string values (non-strings will be +converted to strings with Ruby's `to_s` function) from parsed XML +(using each source field defined using the method above) and place +those values in the destination fields. Configuration: +[source,ruby] +xpath => [ "xpath-syntax", "destination-field" ] + +Values returned by XPath parsing from `xpath-syntax` will be put in the +destination field. Multiple values returned will be pushed onto the +destination field as an array. As such, multiple matches across +multiple source fields will produce duplicate entries in the field. + +[id="{version}-plugins-{type}s-{plugin}-xpath_resources"] +===== Additional XPath resources + +For more information on XPath, see https://www.w3schools.com/xml/xml_xpath.asp. + +The https://www.w3schools.com/xml/xsl_functions.asp[XPath functions] are particularly powerful. + + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/xml-v4.3.0.asciidoc b/docs/versioned-plugins/filters/xml-v4.3.0.asciidoc new file mode 100644 index 000000000..e8d1ebd45 --- /dev/null +++ b/docs/versioned-plugins/filters/xml-v4.3.0.asciidoc @@ -0,0 +1,213 @@ +:plugin: xml +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.3.0 +:release_date: 2025-02-19 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-xml/blob/v4.3.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Xml filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +XML filter. Takes a field that contains XML and expands it into +an actual datastructure. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Xml Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-force_array>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-force_content>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-namespaces>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parse_options>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-remove_namespaces>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-store_xml>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-suppress_empty>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-xpath>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-force_array"] +===== `force_array` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will force single elements to be arrays. Setting this to +false will prevent storing single elements in arrays. + +[id="{version}-plugins-{type}s-{plugin}-force_content"] +===== `force_content` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +By default the filter will expand attributes differently from content inside +of tags. This option allows you to force text content and attributes to always +parse to a hash value. + +[id="{version}-plugins-{type}s-{plugin}-namespaces"] +===== `namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +By default only namespaces declarations on the root element are considered. +This allows to configure all namespace declarations to parse the XML document. + +Example: + +[source,ruby] +filter { + xml { + namespaces => { + "xsl" => "http://www.w3.org/1999/XSL/Transform" + "xhtml" => "http://www.w3.org/1999/xhtml" + } + } +} + +[id="{version}-plugins-{type}s-{plugin}-parse_options"] +===== `parse_options` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Setting XML parse options allows for more control of the parsing process. +By default the parser is not strict and thus accepts some invalid content. +Currently supported options are: + + - `strict` - forces the parser to fail early instead of accumulating errors when content is not valid xml. + +Control characters such as ASCII 0x0 are not allowed and _always_ result in non-valid XML. + +When XML content is not valid, it will be tagged as `_xmlparsefailure`. + +XML specs: + +* XML 1.0 Spec: https://www.w3.org/TR/2008/REC-xml-20081126/#charsets +* XML 1.1 Spec: https://www.w3.org/TR/xml11/#charsets + + + + +[id="{version}-plugins-{type}s-{plugin}-remove_namespaces"] +===== `remove_namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Remove all namespaces from all nodes in the document. +Of course, if the document had nodes with the same names but different namespaces, they will now be ambiguous. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Config for xml to hash is: +[source,ruby] + source => source_field + +For example, if you have the whole XML document in your `message` field: +[source,ruby] + filter { + xml { + source => "message" + } + } + +The above would parse the XML from the `message` field. + +[id="{version}-plugins-{type}s-{plugin}-store_xml"] +===== `store_xml` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will store the whole parsed XML in the destination +field as described above. Setting this to false will prevent that. + +[id="{version}-plugins-{type}s-{plugin}-suppress_empty"] +===== `suppress_empty` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default, output nothing if the element is empty. +If set to `false`, empty element will result in an empty hash object. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define target for placing the data + +For example if you want the data to be put in the `doc` field: +[source,ruby] + filter { + xml { + target => "doc" + } + } + +XML in the value of the source field will be expanded into a +datastructure in the `target` field. +Note: if the `target` field already exists, it will be overridden. +Required if `store_xml` is true (which is the default). + +[id="{version}-plugins-{type}s-{plugin}-xpath"] +===== `xpath` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +xpath will additionally select string values (non-strings will be +converted to strings with Ruby's `to_s` function) from parsed XML +(using each source field defined using the method above) and place +those values in the destination fields. Configuration: +[source,ruby] +xpath => [ "xpath-syntax", "destination-field" ] + +Values returned by XPath parsing from `xpath-syntax` will be put in the +destination field. Multiple values returned will be pushed onto the +destination field as an array. As such, multiple matches across +multiple source fields will produce duplicate entries in the field. + +[id="{version}-plugins-{type}s-{plugin}-xpath_resources"] +===== Additional XPath resources + +For more information on XPath, see https://www.w3schools.com/xml/xml_xpath.asp. + +The https://www.w3schools.com/xml/xsl_functions.asp[XPath functions] are particularly powerful. + + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/xml-v4.3.1.asciidoc b/docs/versioned-plugins/filters/xml-v4.3.1.asciidoc new file mode 100644 index 000000000..648cd6a1a --- /dev/null +++ b/docs/versioned-plugins/filters/xml-v4.3.1.asciidoc @@ -0,0 +1,213 @@ +:plugin: xml +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.3.1 +:release_date: 2025-04-22 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-xml/blob/v4.3.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Xml filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +XML filter. Takes a field that contains XML and expands it into +an actual datastructure. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Xml Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-force_array>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-force_content>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-namespaces>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parse_options>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-remove_namespaces>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-store_xml>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-suppress_empty>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-xpath>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-force_array"] +===== `force_array` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will force single elements to be arrays. Setting this to +false will prevent storing single elements in arrays. + +[id="{version}-plugins-{type}s-{plugin}-force_content"] +===== `force_content` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +By default the filter will expand attributes differently from content inside +of tags. This option allows you to force text content and attributes to always +parse to a hash value. + +[id="{version}-plugins-{type}s-{plugin}-namespaces"] +===== `namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +By default only namespaces declarations on the root element are considered. +This allows to configure all namespace declarations to parse the XML document. + +Example: + +[source,ruby] +filter { + xml { + namespaces => { + "xsl" => "http://www.w3.org/1999/XSL/Transform" + "xhtml" => "http://www.w3.org/1999/xhtml" + } + } +} + +[id="{version}-plugins-{type}s-{plugin}-parse_options"] +===== `parse_options` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Setting XML parse options allows for more control of the parsing process. +By default the parser is not strict and thus accepts some invalid content. +Currently supported options are: + + - `strict` - forces the parser to fail early instead of accumulating errors when content is not valid xml. + +Control characters such as ASCII 0x0 are not allowed and _always_ result in non-valid XML. + +When XML content is not valid, it will be tagged as `_xmlparsefailure`. + +XML specs: + +* XML 1.0 Spec: https://www.w3.org/TR/2008/REC-xml-20081126/#charsets +* XML 1.1 Spec: https://www.w3.org/TR/xml11/#charsets + + + + +[id="{version}-plugins-{type}s-{plugin}-remove_namespaces"] +===== `remove_namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Remove all namespaces from all nodes in the document. +Of course, if the document had nodes with the same names but different namespaces, they will now be ambiguous. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Config for xml to hash is: +[source,ruby] + source => source_field + +For example, if you have the whole XML document in your `message` field: +[source,ruby] + filter { + xml { + source => "message" + } + } + +The above would parse the XML from the `message` field. + +[id="{version}-plugins-{type}s-{plugin}-store_xml"] +===== `store_xml` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will store the whole parsed XML in the destination +field as described above. Setting this to false will prevent that. + +[id="{version}-plugins-{type}s-{plugin}-suppress_empty"] +===== `suppress_empty` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default, output nothing if the element is empty. +If set to `false`, empty element will result in an empty hash object. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define target for placing the data + +For example if you want the data to be put in the `doc` field: +[source,ruby] + filter { + xml { + target => "doc" + } + } + +XML in the value of the source field will be expanded into a +datastructure in the `target` field. +Note: if the `target` field already exists, it will be overridden. +Required if `store_xml` is true (which is the default). + +[id="{version}-plugins-{type}s-{plugin}-xpath"] +===== `xpath` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +xpath will additionally select string values (non-strings will be +converted to strings with Ruby's `to_s` function) from parsed XML +(using each source field defined using the method above) and place +those values in the destination fields. Configuration: +[source,ruby] +xpath => [ "xpath-syntax", "destination-field" ] + +Values returned by XPath parsing from `xpath-syntax` will be put in the +destination field. Multiple values returned will be pushed onto the +destination field as an array. As such, multiple matches across +multiple source fields will produce duplicate entries in the field. + +[id="{version}-plugins-{type}s-{plugin}-xpath_resources"] +===== Additional XPath resources + +For more information on XPath, see https://www.w3schools.com/xml/xml_xpath.asp. + +The https://www.w3schools.com/xml/xsl_functions.asp[XPath functions] are particularly powerful. + + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/xml-v4.3.2.asciidoc b/docs/versioned-plugins/filters/xml-v4.3.2.asciidoc new file mode 100644 index 000000000..491a1e29e --- /dev/null +++ b/docs/versioned-plugins/filters/xml-v4.3.2.asciidoc @@ -0,0 +1,213 @@ +:plugin: xml +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.3.2 +:release_date: 2025-07-24 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-xml/blob/v4.3.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Xml filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +XML filter. Takes a field that contains XML and expands it into +an actual datastructure. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Xml Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-force_array>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-force_content>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-namespaces>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parse_options>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-remove_namespaces>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-store_xml>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-suppress_empty>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-xpath>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-force_array"] +===== `force_array` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will force single elements to be arrays. Setting this to +false will prevent storing single elements in arrays. + +[id="{version}-plugins-{type}s-{plugin}-force_content"] +===== `force_content` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +By default the filter will expand attributes differently from content inside +of tags. This option allows you to force text content and attributes to always +parse to a hash value. + +[id="{version}-plugins-{type}s-{plugin}-namespaces"] +===== `namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +By default only namespaces declarations on the root element are considered. +This allows to configure all namespace declarations to parse the XML document. + +Example: + +[source,ruby] +filter { + xml { + namespaces => { + "xsl" => "http://www.w3.org/1999/XSL/Transform" + "xhtml" => "http://www.w3.org/1999/xhtml" + } + } +} + +[id="{version}-plugins-{type}s-{plugin}-parse_options"] +===== `parse_options` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Setting XML parse options allows for more control of the parsing process. +By default the parser is not strict and thus accepts some invalid content. +Currently supported options are: + + - `strict` - forces the parser to fail early instead of accumulating errors when content is not valid xml. + +Control characters such as ASCII 0x0 are not allowed and _always_ result in non-valid XML. + +When XML content is not valid, it will be tagged as `_xmlparsefailure`. + +XML specs: + +* XML 1.0 Spec: https://www.w3.org/TR/2008/REC-xml-20081126/#charsets +* XML 1.1 Spec: https://www.w3.org/TR/xml11/#charsets + + + + +[id="{version}-plugins-{type}s-{plugin}-remove_namespaces"] +===== `remove_namespaces` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Remove all namespaces from all nodes in the document. +Of course, if the document had nodes with the same names but different namespaces, they will now be ambiguous. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Config for xml to hash is: +[source,ruby] + source => source_field + +For example, if you have the whole XML document in your `message` field: +[source,ruby] + filter { + xml { + source => "message" + } + } + +The above would parse the XML from the `message` field. + +[id="{version}-plugins-{type}s-{plugin}-store_xml"] +===== `store_xml` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default the filter will store the whole parsed XML in the destination +field as described above. Setting this to false will prevent that. + +[id="{version}-plugins-{type}s-{plugin}-suppress_empty"] +===== `suppress_empty` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +By default, output nothing if the element is empty. +If set to `false`, empty element will result in an empty hash object. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define target for placing the data + +For example if you want the data to be put in the `doc` field: +[source,ruby] + filter { + xml { + target => "doc" + } + } + +XML in the value of the source field will be expanded into a +datastructure in the `target` field. +Note: if the `target` field already exists, it will be overridden. +Required if `store_xml` is true (which is the default). + +[id="{version}-plugins-{type}s-{plugin}-xpath"] +===== `xpath` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +xpath will additionally select string values (non-strings will be +converted to strings with Ruby's `to_s` function) from parsed XML +(using each source field defined using the method above) and place +those values in the destination fields. Configuration: +[source,ruby] +xpath => [ "xpath-syntax", "destination-field" ] + +Values returned by XPath parsing from `xpath-syntax` will be put in the +destination field. Multiple values returned will be pushed onto the +destination field as an array. As such, multiple matches across +multiple source fields will produce duplicate entries in the field. + +[id="{version}-plugins-{type}s-{plugin}-xpath_resources"] +===== Additional XPath resources + +For more information on XPath, see https://www.w3schools.com/xml/xml_xpath.asp. + +The https://www.w3schools.com/xml/xsl_functions.asp[XPath functions] are particularly powerful. + + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-index.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-index.asciidoc index 6dbc35fda..2622262d5 100644 --- a/docs/versioned-plugins/inputs/azure_event_hubs-index.asciidoc +++ b/docs/versioned-plugins/inputs/azure_event_hubs-index.asciidoc @@ -5,6 +5,13 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-07-11 +| <> | 2025-01-03 +| <> | 2024-10-25 +| <> | 2024-10-02 +| <> | 2024-10-02 +| <> | 2024-06-07 +| <> | 2024-05-29 | <> | 2023-07-04 | <> | 2022-06-13 | <> | 2022-01-04 @@ -27,6 +34,13 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2018-06-26 |======================================================================= +include::azure_event_hubs-v1.5.2.asciidoc[] +include::azure_event_hubs-v1.5.1.asciidoc[] +include::azure_event_hubs-v1.5.0.asciidoc[] +include::azure_event_hubs-v1.4.9.asciidoc[] +include::azure_event_hubs-v1.4.8.asciidoc[] +include::azure_event_hubs-v1.4.7.asciidoc[] +include::azure_event_hubs-v1.4.6.asciidoc[] include::azure_event_hubs-v1.4.5.asciidoc[] include::azure_event_hubs-v1.4.4.asciidoc[] include::azure_event_hubs-v1.4.3.asciidoc[] diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.6.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.6.asciidoc new file mode 100644 index 000000000..2743ff7a7 --- /dev/null +++ b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.6.asciidoc @@ -0,0 +1,525 @@ +:plugin: azure_event_hubs +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.4.6 +:release_date: 2024-05-29 +:changelog_url: https://github.com/logstash-plugins/logstash-input-azure_event_hubs/blob/v1.4.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Azure Event Hubs plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This plugin consumes events from +https://azure.microsoft.com/en-us/services/event-hubs[Azure Event Hubs], a +highly scalable data streaming platform and event ingestion service. Event +producers send events to the Azure Event Hub, and this plugin consumes those +events for use with Logstash. + +Many Azure services integrate with the Azure Event Hubs. +https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Azure +Monitor], for example, integrates with Azure Event Hubs to provide infrastructure metrics. + +IMPORTANT: This plugin requires outbound connections to ports `tcp/443`, `tcp/9093`, `tcp/5671`, and `tcp/5672`, +as noted in the https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall[Microsoft Event Hub documentation]. + +===== Event Hub connection string + +The plugin uses the connection string to access Azure Events Hubs. Find the +connection string here: https://portal.azure.com[Azure Portal]`-> Event Hub -> +Shared access polices`. The event_hub_connections option passes the Event Hub +connection strings for the basic configuration. + +Sample connection string: + +[source,text] +---- +Endpoint=sb://logstash.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs +---- + +===== Blob Storage and connection string + +https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage +account] is an essential part of Azure-to-Logstash configuration. +A Blob Storage account is a central location that enables multiple instances of +Logstash to work together to process events. It records the +offset (location) of processed events. On restart, Logstash resumes processing +exactly where it left off. + +Configuration notes: + +* A Blob Storage account is highly recommended for use with this plugin, and is +likely required for production servers. +* The `storage_connection` option passes the blob storage connection string. +* Configure all Logstash instances to use the same `storage_connection` to get the +benefits of shared processing. + +Sample Blob Storage connection string: + +[source,text] +---- +DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net +---- + +Find the connection string to Blob Storage here: +https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. + +[id="{version}-plugins-{type}s-{plugin}-best-practices"] +===== Best practices + +Here are some guidelines to help you avoid data conflicts that can cause lost +events. + +* <<{version}-plugins-{type}s-{plugin}-bp-group>> +* <<{version}-plugins-{type}s-{plugin}-bp-multihub>> +* <<{version}-plugins-{type}s-{plugin}-bp-threads>> + +[id="{version}-plugins-{type}s-{plugin}-bp-group"] +===== Create a Logstash consumer group +Create a new consumer group specifically for Logstash. Do not use the $default or +any other consumer group that might already be in use. Reusing consumer groups +among non-related consumers can cause unexpected behavior and possibly lost +events. All Logstash instances should use the same consumer group so that they can +work together for processing events. + +[id="{version}-plugins-{type}s-{plugin}-bp-multihub"] +===== Avoid overwriting offset with multiple Event Hubs +The offsets (position) of the Event Hubs are stored in the configured Azure Blob +store. The Azure Blob store uses paths like a file system to store the offsets. +If the paths between multiple Event Hubs overlap, then the offsets may be stored +incorrectly. + +To avoid duplicate file paths, use the advanced configuration model and make +sure that at least one of these options is different per Event Hub: + +** storage_connection +** storage_container (defaults to Event Hub name if not defined) +** consumer_group + + +[id="{version}-plugins-{type}s-{plugin}-bp-threads"] +===== Set number of threads correctly + +By default, the number of threads used to service all event hubs is `16`. And while this +may be sufficient for most use cases, throughput may be improved by refining this number. +When servicing a large number of partitions across one or more event hubs, setting a higher +value may result in improved performance. The maximum number of threads is not strictly bound +by the total number of partitions being serviced, but setting the value much higher than +that may mean that some threads are idle. + +NOTE: The number of threads *must* be greater than or equal to the number of Event hubs plus one. + +NOTE: Threads are currently available only as a global setting across all event hubs in a single `azure_event_hubs` +input definition. However if your configuration includes multiple `azure_event_hubs` inputs, the threads setting applies +independently to each. + +**Sample scenarios** + +* Event Hubs = 4. Partitions on each Event Hub = 3. +Minimum threads is 5 (4 Event Hubs plus one). +* If you’re collecting activity logs from one event hub instance, +then only 2 threads (1 Event Hub plus one) are required. + + +[id="{version}-plugins-{type}s-{plugin}-eh_config_models"] +==== Configuration models + +This plugin supports two configuration models: basic and advanced. Basic +configuration is recommended for most use cases, and is illustrated in the +examples throughout this topic. + +[id="{version}-plugins-{type}s-{plugin}-eh_basic_config"] +===== Basic configuration (default) + +Basic configuration is the default and supports consuming from multiple Event +Hubs. All Events Hubs, except for the connection string, share the same +configuration. + +You supply a list of Event Hub connection strings, complete with the Event Hub +EntityPath that defines the Event Hub name. All other configuration settings are +shared. + +[source,ruby] +---- +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...EntityPath=insights-logs-errors", "Endpoint=sb://example2...EntityPath=insights-metrics-pt1m"] + threads => 8 + decorate_events => true + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-eh_advanced_config"] +===== Advanced configuration +The advanced configuration model accommodates deployments where different Event +Hubs require different configurations. Options can be configured per Event Hub. +You provide a list of Event Hub names through the `event_hubs` option. Under +each name, specify the configuration for that Event Hub. Options can be defined +globally or expressed per Event Hub. + +If the same configuration option appears in both the global and `event_hub` +sections, the more specific (event_hub) setting takes precedence. + +NOTE: Advanced configuration is not necessary or recommended for most use cases. + +[source,ruby] +---- +input { + azure_event_hubs { + config_mode => "advanced" + threads => 8 + decorate_events => true + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + initial_position => "beginning" + consumer_group => "iam_team" + }}, + {"insights-metrics-pt1m" => { + event_hub_connection => "Endpoint=sb://example2..." + initial_position => "end" + consumer_group => "db_team" + }} + ] + } +} +---- + +In this example, `storage_connection` and `decorate_events` are applied globally. +The two Event Hubs each have their own settings for `consumer_groups` +and `initial_position`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Azure Event Hubs Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-config_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], (`basic` or `advanced`)|No +| <<{version}-plugins-{type}s-{plugin}-event_hubs>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connections>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => basic` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-checkpoint_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_group>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-initial_position>> |{logstash-ref}/configuration-file-structure.html#string[string], (`beginning`, `end`, or `look_back`)|No +| <<{version}-plugins-{type}s-{plugin}-initial_position_look_back>> |{logstash-ref}/configuration-file-structure.html#number[number]|No, unless `initial_position => look_back` +| <<{version}-plugins-{type}s-{plugin}-max_batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-storage_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_container>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +NOTE: All Event Hubs options are common to both basic and advanced +configurations, with the following exceptions. The basic configuration uses +`event_hub_connections` to support multiple connections. The advanced +configuration uses `event_hubs` and `event_hub_connection` (singular). + +[id="{version}-plugins-{type}s-{plugin}-config_mode"] +===== `config_mode` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid entries are `basic` or `advanced` +* Default value is `basic` + +Sets configuration to either <<{version}-plugins-{type}s-{plugin}-eh_basic_config>> or <<{version}-plugins-{type}s-{plugin}-eh_advanced_config>>. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hubs"] +===== `event_hubs` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Ignored for basic configuration +* Required for advanced configuration + +Defines the Event Hubs to be read. An array of hashes where each entry is a +hash of the Event Hub name and its configuration options. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1..." + }}, + { "event_hub_name2" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" + }} + ] + consumer_group => "logstash" # shared across all Event Hubs +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connections"] +===== `event_hub_connections` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Required for basic configuration + +List of connection strings that identifies the Event Hubs to be read. Connection +strings include the EntityPath for the Event Hub. + +The `event_hub_connections` option is defined +per Event Hub. All other configuration options are shared among Event Hubs. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connection"] +===== `event_hub_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value +* Valid only for advanced configuration + +Connection string that identifies the Event Hub to be read. Advanced +configuration options can be set per Event Hub. This option modifies +`event_hub_name`, and should be nested under it. (See sample.) This option +accepts only one connection string. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1...;EntityPath=event_hub_name1" + }} + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-checkpoint_interval"] +===== `checkpoint_interval` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `5` seconds +* Set to `0` to disable. + +Interval in seconds to write checkpoints during batch processing. Checkpoints +tell Logstash where to resume processing after a restart. Checkpoints are +automatically written at the end of each batch, regardless of this setting. + +Writing checkpoints too frequently can slow down processing unnecessarily. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + checkpoint_interval => 5 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-consumer_group"] +===== `consumer_group` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `$Default` + +Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that +consumer group so that they can work together properly. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + consumer_group => "logstash" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Adds metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + decorate_events => true +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position"] +===== `initial_position` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid arguments are `beginning`, `end`, `look_back` +* Default value is `beginning` + +When first reading from an Event Hub, start from this position: + +* `beginning` reads all pre-existing events in the Event Hub +* `end` does not read any pre-existing events in the Event Hub +* `look_back` reads `end` minus a number of seconds worth of pre-existing events. +You control the number of seconds using the `initial_position_look_back` option. + +Note: If `storage_connection` is set, the `initial_position` value is used only +the first time Logstash reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "beginning" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position_look_back"] +===== `initial_position_look_back` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `86400` +* Used only if `initial_position` is set to `look-back` + +Number of seconds to look back to find the initial position for pre-existing +events. This option is used only if `initial_position` is set to `look_back`. If +`storage_connection` is set, this configuration applies only the first time Logstash +reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "look_back" + initial_position_look_back => 86400 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-max_batch_size"] +===== `max_batch_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `125` + +Maximum number of events retrieved and processed together. A checkpoint is +created after each batch. Increasing this value may help with performance, but +requires more memory. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + max_batch_size => 125 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_connection"] +===== `storage_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value + +Connection string for blob account storage. Blob account storage persists the +offsets between restarts, and ensures that multiple instances of Logstash +process different partitions. +When this value is set, restarts resume where processing left off. +When this value is not set, the `initial_position` value is used on every restart. + +We strongly recommend that you define this value for production environments. + + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_container"] +===== `storage_container` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Defaults to the Event Hub name if not defined + +Name of the storage container used to persist offsets and allow multiple instances of Logstash +to work together. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" +} +---- + +To avoid overwriting offsets, you can use different storage containers. This is +particularly important if you are monitoring two Event Hubs with the same name. +You can use the advanced configuration model to configure different storage +containers. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + storage_container => "insights-operational-logs-1" + }}, + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_container => "insights-operational-logs-2" + }} + ] + } +---- + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Minimum value is `2` +* Default value is `16` + +Total number of threads used to process events. The value you set here applies +to all Event Hubs. Even with advanced configuration, this value is a global +setting, and can't be set per event hub. + +[source,ruby] +---- +azure_event_hubs { + threads => 16 +} +---- + +The number of threads should be the number of Event Hubs plus one or more. +See <<{version}-plugins-{type}s-{plugin}-best-practices>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.7.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.7.asciidoc new file mode 100644 index 000000000..b72b74f08 --- /dev/null +++ b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.7.asciidoc @@ -0,0 +1,536 @@ +:plugin: azure_event_hubs +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.4.7 +:release_date: 2024-06-07 +:changelog_url: https://github.com/logstash-plugins/logstash-input-azure_event_hubs/blob/v1.4.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Azure Event Hubs plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This plugin consumes events from +https://azure.microsoft.com/en-us/services/event-hubs[Azure Event Hubs], a +highly scalable data streaming platform and event ingestion service. Event +producers send events to the Azure Event Hub, and this plugin consumes those +events for use with Logstash. + +Many Azure services integrate with the Azure Event Hubs. +https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Azure +Monitor], for example, integrates with Azure Event Hubs to provide infrastructure metrics. + +IMPORTANT: This plugin requires outbound connections to ports `tcp/443`, `tcp/9093`, `tcp/5671`, and `tcp/5672`, +as noted in the https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall[Microsoft Event Hub documentation]. + +===== Event Hub connection string + +The plugin uses the connection string to access Azure Events Hubs. Find the +connection string here: https://portal.azure.com[Azure Portal]`-> Event Hub -> +Shared access polices`. The event_hub_connections option passes the Event Hub +connection strings for the basic configuration. + +Sample connection string: + +[source,text] +---- +Endpoint=sb://logstash.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs +---- + +===== Blob Storage and connection string + +https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage +account] is an essential part of Azure-to-Logstash configuration. +A Blob Storage account is a central location that enables multiple instances of +Logstash to work together to process events. It records the +offset (location) of processed events. On restart, Logstash resumes processing +exactly where it left off. + +Configuration notes: + +* A Blob Storage account is highly recommended for use with this plugin, and is +likely required for production servers. +* The `storage_connection` option passes the blob storage connection string. +* Configure all Logstash instances to use the same `storage_connection` to get the +benefits of shared processing. + +Sample Blob Storage connection string: + +[source,text] +---- +DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net +---- + +Find the connection string to Blob Storage here: +https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. + +[id="{version}-plugins-{type}s-{plugin}-best-practices"] +===== Best practices + +Here are some guidelines to help you avoid data conflicts that can cause lost +events. + +* <<{version}-plugins-{type}s-{plugin}-bp-group>> +* <<{version}-plugins-{type}s-{plugin}-bp-multihub>> +* <<{version}-plugins-{type}s-{plugin}-bp-threads>> + +[id="{version}-plugins-{type}s-{plugin}-bp-group"] +===== Create a Logstash consumer group +Create a new consumer group specifically for Logstash. Do not use the $default or +any other consumer group that might already be in use. Reusing consumer groups +among non-related consumers can cause unexpected behavior and possibly lost +events. All Logstash instances should use the same consumer group so that they can +work together for processing events. + +[id="{version}-plugins-{type}s-{plugin}-bp-multihub"] +===== Avoid overwriting offset with multiple Event Hubs +The offsets (position) of the Event Hubs are stored in the configured Azure Blob +store. The Azure Blob store uses paths like a file system to store the offsets. +If the paths between multiple Event Hubs overlap, then the offsets may be stored +incorrectly. + +To avoid duplicate file paths, use the advanced configuration model and make +sure that at least one of these options is different per Event Hub: + +** storage_connection +** storage_container (defaults to Event Hub name if not defined) +** consumer_group + + +[id="{version}-plugins-{type}s-{plugin}-bp-threads"] +===== Set number of threads correctly + +By default, the number of threads used to service all event hubs is `16`. And while this +may be sufficient for most use cases, throughput may be improved by refining this number. +When servicing a large number of partitions across one or more event hubs, setting a higher +value may result in improved performance. The maximum number of threads is not strictly bound +by the total number of partitions being serviced, but setting the value much higher than +that may mean that some threads are idle. + +NOTE: The number of threads *must* be greater than or equal to the number of Event hubs plus one. + +NOTE: Threads are currently available only as a global setting across all event hubs in a single `azure_event_hubs` +input definition. However if your configuration includes multiple `azure_event_hubs` inputs, the threads setting applies +independently to each. + +**Example: Single event hub** + +If you’re collecting activity logs from one event hub instance, +then only 2 threads are required. + +* Event hubs = 1 +* Minimum threads = 2 (1 Event Hub + 1) + +**Example: Multiple event hubs** + +If you are collecting activity logs from more than event hub instance, then at least 1 thread per event hub is required. + +* Event hubs = 4 +* Minimum threads = 5 (4 Event Hubs + 1) + +When you are using multiple partitions per event hub, you may want to assign more threads. +A good base level is (1 + `number of event hubs * number of partitions`). +That is, one thread for each partition across all event hubs. + +[id="{version}-plugins-{type}s-{plugin}-eh_config_models"] +==== Configuration models + +This plugin supports two configuration models: basic and advanced. Basic +configuration is recommended for most use cases, and is illustrated in the +examples throughout this topic. + +[id="{version}-plugins-{type}s-{plugin}-eh_basic_config"] +===== Basic configuration (default) + +Basic configuration is the default and supports consuming from multiple Event +Hubs. All Events Hubs, except for the connection string, share the same +configuration. + +You supply a list of Event Hub connection strings, complete with the Event Hub +EntityPath that defines the Event Hub name. All other configuration settings are +shared. + +[source,ruby] +---- +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...EntityPath=insights-logs-errors", "Endpoint=sb://example2...EntityPath=insights-metrics-pt1m"] + threads => 8 + decorate_events => true + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-eh_advanced_config"] +===== Advanced configuration +The advanced configuration model accommodates deployments where different Event +Hubs require different configurations. Options can be configured per Event Hub. +You provide a list of Event Hub names through the `event_hubs` option. Under +each name, specify the configuration for that Event Hub. Options can be defined +globally or expressed per Event Hub. + +If the same configuration option appears in both the global and `event_hub` +sections, the more specific (event_hub) setting takes precedence. + +NOTE: Advanced configuration is not necessary or recommended for most use cases. + +[source,ruby] +---- +input { + azure_event_hubs { + config_mode => "advanced" + threads => 8 + decorate_events => true + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + initial_position => "beginning" + consumer_group => "iam_team" + }}, + {"insights-metrics-pt1m" => { + event_hub_connection => "Endpoint=sb://example2..." + initial_position => "end" + consumer_group => "db_team" + }} + ] + } +} +---- + +In this example, `storage_connection` and `decorate_events` are applied globally. +The two Event Hubs each have their own settings for `consumer_groups` +and `initial_position`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Azure Event Hubs Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-config_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], (`basic` or `advanced`)|No +| <<{version}-plugins-{type}s-{plugin}-event_hubs>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connections>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => basic` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-checkpoint_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_group>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-initial_position>> |{logstash-ref}/configuration-file-structure.html#string[string], (`beginning`, `end`, or `look_back`)|No +| <<{version}-plugins-{type}s-{plugin}-initial_position_look_back>> |{logstash-ref}/configuration-file-structure.html#number[number]|No, unless `initial_position => look_back` +| <<{version}-plugins-{type}s-{plugin}-max_batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-storage_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_container>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +NOTE: All Event Hubs options are common to both basic and advanced +configurations, with the following exceptions. The basic configuration uses +`event_hub_connections` to support multiple connections. The advanced +configuration uses `event_hubs` and `event_hub_connection` (singular). + +[id="{version}-plugins-{type}s-{plugin}-config_mode"] +===== `config_mode` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid entries are `basic` or `advanced` +* Default value is `basic` + +Sets configuration to either <<{version}-plugins-{type}s-{plugin}-eh_basic_config>> or <<{version}-plugins-{type}s-{plugin}-eh_advanced_config>>. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hubs"] +===== `event_hubs` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Ignored for basic configuration +* Required for advanced configuration + +Defines the Event Hubs to be read. An array of hashes where each entry is a +hash of the Event Hub name and its configuration options. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1..." + }}, + { "event_hub_name2" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" + }} + ] + consumer_group => "logstash" # shared across all Event Hubs +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connections"] +===== `event_hub_connections` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Required for basic configuration + +List of connection strings that identifies the Event Hubs to be read. Connection +strings include the EntityPath for the Event Hub. + +The `event_hub_connections` option is defined +per Event Hub. All other configuration options are shared among Event Hubs. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connection"] +===== `event_hub_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value +* Valid only for advanced configuration + +Connection string that identifies the Event Hub to be read. Advanced +configuration options can be set per Event Hub. This option modifies +`event_hub_name`, and should be nested under it. (See sample.) This option +accepts only one connection string. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1...;EntityPath=event_hub_name1" + }} + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-checkpoint_interval"] +===== `checkpoint_interval` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `5` seconds +* Set to `0` to disable. + +Interval in seconds to write checkpoints during batch processing. Checkpoints +tell Logstash where to resume processing after a restart. Checkpoints are +automatically written at the end of each batch, regardless of this setting. + +Writing checkpoints too frequently can slow down processing unnecessarily. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + checkpoint_interval => 5 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-consumer_group"] +===== `consumer_group` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `$Default` + +Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that +consumer group so that they can work together properly. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + consumer_group => "logstash" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Adds metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + decorate_events => true +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position"] +===== `initial_position` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid arguments are `beginning`, `end`, `look_back` +* Default value is `beginning` + +When first reading from an Event Hub, start from this position: + +* `beginning` reads all pre-existing events in the Event Hub +* `end` does not read any pre-existing events in the Event Hub +* `look_back` reads `end` minus a number of seconds worth of pre-existing events. +You control the number of seconds using the `initial_position_look_back` option. + +Note: If `storage_connection` is set, the `initial_position` value is used only +the first time Logstash reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "beginning" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position_look_back"] +===== `initial_position_look_back` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `86400` +* Used only if `initial_position` is set to `look-back` + +Number of seconds to look back to find the initial position for pre-existing +events. This option is used only if `initial_position` is set to `look_back`. If +`storage_connection` is set, this configuration applies only the first time Logstash +reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "look_back" + initial_position_look_back => 86400 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-max_batch_size"] +===== `max_batch_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `125` + +Maximum number of events retrieved and processed together. A checkpoint is +created after each batch. Increasing this value may help with performance, but +requires more memory. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + max_batch_size => 125 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_connection"] +===== `storage_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value + +Connection string for blob account storage. Blob account storage persists the +offsets between restarts, and ensures that multiple instances of Logstash +process different partitions. +When this value is set, restarts resume where processing left off. +When this value is not set, the `initial_position` value is used on every restart. + +We strongly recommend that you define this value for production environments. + + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_container"] +===== `storage_container` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Defaults to the Event Hub name if not defined + +Name of the storage container used to persist offsets and allow multiple instances of Logstash +to work together. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" +} +---- + +To avoid overwriting offsets, you can use different storage containers. This is +particularly important if you are monitoring two Event Hubs with the same name. +You can use the advanced configuration model to configure different storage +containers. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + storage_container => "insights-operational-logs-1" + }}, + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_container => "insights-operational-logs-2" + }} + ] + } +---- + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Minimum value is `2` +* Default value is `16` + +Total number of threads used to process events. The value you set here applies +to all Event Hubs. Even with advanced configuration, this value is a global +setting, and can't be set per event hub. + +[source,ruby] +---- +azure_event_hubs { + threads => 16 +} +---- + +The number of threads should be the number of Event Hubs plus one or more. +See <<{version}-plugins-{type}s-{plugin}-best-practices>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.8.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.8.asciidoc new file mode 100644 index 000000000..94ddf0d6a --- /dev/null +++ b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.8.asciidoc @@ -0,0 +1,536 @@ +:plugin: azure_event_hubs +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.4.8 +:release_date: 2024-10-02 +:changelog_url: https://github.com/logstash-plugins/logstash-input-azure_event_hubs/blob/v1.4.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Azure Event Hubs plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This plugin consumes events from +https://azure.microsoft.com/en-us/services/event-hubs[Azure Event Hubs], a +highly scalable data streaming platform and event ingestion service. Event +producers send events to the Azure Event Hub, and this plugin consumes those +events for use with Logstash. + +Many Azure services integrate with the Azure Event Hubs. +https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Azure +Monitor], for example, integrates with Azure Event Hubs to provide infrastructure metrics. + +IMPORTANT: This plugin requires outbound connections to ports `tcp/443`, `tcp/9093`, `tcp/5671`, and `tcp/5672`, +as noted in the https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall[Microsoft Event Hub documentation]. + +===== Event Hub connection string + +The plugin uses the connection string to access Azure Events Hubs. Find the +connection string here: https://portal.azure.com[Azure Portal]`-> Event Hub -> +Shared access polices`. The event_hub_connections option passes the Event Hub +connection strings for the basic configuration. + +Sample connection string: + +[source,text] +---- +Endpoint=sb://logstash.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs +---- + +===== Blob Storage and connection string + +https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage +account] is an essential part of Azure-to-Logstash configuration. +A Blob Storage account is a central location that enables multiple instances of +Logstash to work together to process events. It records the +offset (location) of processed events. On restart, Logstash resumes processing +exactly where it left off. + +Configuration notes: + +* A Blob Storage account is highly recommended for use with this plugin, and is +likely required for production servers. +* The `storage_connection` option passes the blob storage connection string. +* Configure all Logstash instances to use the same `storage_connection` to get the +benefits of shared processing. + +Sample Blob Storage connection string: + +[source,text] +---- +DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net +---- + +Find the connection string to Blob Storage here: +https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. + +[id="{version}-plugins-{type}s-{plugin}-best-practices"] +===== Best practices + +Here are some guidelines to help you avoid data conflicts that can cause lost +events. + +* <<{version}-plugins-{type}s-{plugin}-bp-group>> +* <<{version}-plugins-{type}s-{plugin}-bp-multihub>> +* <<{version}-plugins-{type}s-{plugin}-bp-threads>> + +[id="{version}-plugins-{type}s-{plugin}-bp-group"] +===== Create a Logstash consumer group +Create a new consumer group specifically for Logstash. Do not use the $default or +any other consumer group that might already be in use. Reusing consumer groups +among non-related consumers can cause unexpected behavior and possibly lost +events. All Logstash instances should use the same consumer group so that they can +work together for processing events. + +[id="{version}-plugins-{type}s-{plugin}-bp-multihub"] +===== Avoid overwriting offset with multiple Event Hubs +The offsets (position) of the Event Hubs are stored in the configured Azure Blob +store. The Azure Blob store uses paths like a file system to store the offsets. +If the paths between multiple Event Hubs overlap, then the offsets may be stored +incorrectly. + +To avoid duplicate file paths, use the advanced configuration model and make +sure that at least one of these options is different per Event Hub: + +** storage_connection +** storage_container (defaults to Event Hub name if not defined) +** consumer_group + + +[id="{version}-plugins-{type}s-{plugin}-bp-threads"] +===== Set number of threads correctly + +By default, the number of threads used to service all event hubs is `16`. And while this +may be sufficient for most use cases, throughput may be improved by refining this number. +When servicing a large number of partitions across one or more event hubs, setting a higher +value may result in improved performance. The maximum number of threads is not strictly bound +by the total number of partitions being serviced, but setting the value much higher than +that may mean that some threads are idle. + +NOTE: The number of threads *must* be greater than or equal to the number of Event hubs plus one. + +NOTE: Threads are currently available only as a global setting across all event hubs in a single `azure_event_hubs` +input definition. However if your configuration includes multiple `azure_event_hubs` inputs, the threads setting applies +independently to each. + +**Example: Single event hub** + +If you’re collecting activity logs from one event hub instance, +then only 2 threads are required. + +* Event hubs = 1 +* Minimum threads = 2 (1 Event Hub + 1) + +**Example: Multiple event hubs** + +If you are collecting activity logs from more than event hub instance, then at least 1 thread per event hub is required. + +* Event hubs = 4 +* Minimum threads = 5 (4 Event Hubs + 1) + +When you are using multiple partitions per event hub, you may want to assign more threads. +A good base level is (1 + `number of event hubs * number of partitions`). +That is, one thread for each partition across all event hubs. + +[id="{version}-plugins-{type}s-{plugin}-eh_config_models"] +==== Configuration models + +This plugin supports two configuration models: basic and advanced. Basic +configuration is recommended for most use cases, and is illustrated in the +examples throughout this topic. + +[id="{version}-plugins-{type}s-{plugin}-eh_basic_config"] +===== Basic configuration (default) + +Basic configuration is the default and supports consuming from multiple Event +Hubs. All Events Hubs, except for the connection string, share the same +configuration. + +You supply a list of Event Hub connection strings, complete with the Event Hub +EntityPath that defines the Event Hub name. All other configuration settings are +shared. + +[source,ruby] +---- +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...EntityPath=insights-logs-errors", "Endpoint=sb://example2...EntityPath=insights-metrics-pt1m"] + threads => 8 + decorate_events => true + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-eh_advanced_config"] +===== Advanced configuration +The advanced configuration model accommodates deployments where different Event +Hubs require different configurations. Options can be configured per Event Hub. +You provide a list of Event Hub names through the `event_hubs` option. Under +each name, specify the configuration for that Event Hub. Options can be defined +globally or expressed per Event Hub. + +If the same configuration option appears in both the global and `event_hub` +sections, the more specific (event_hub) setting takes precedence. + +NOTE: Advanced configuration is not necessary or recommended for most use cases. + +[source,ruby] +---- +input { + azure_event_hubs { + config_mode => "advanced" + threads => 8 + decorate_events => true + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + initial_position => "beginning" + consumer_group => "iam_team" + }}, + {"insights-metrics-pt1m" => { + event_hub_connection => "Endpoint=sb://example2..." + initial_position => "end" + consumer_group => "db_team" + }} + ] + } +} +---- + +In this example, `storage_connection` and `decorate_events` are applied globally. +The two Event Hubs each have their own settings for `consumer_groups` +and `initial_position`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Azure Event Hubs Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-config_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], (`basic` or `advanced`)|No +| <<{version}-plugins-{type}s-{plugin}-event_hubs>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connections>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => basic` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-checkpoint_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_group>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-initial_position>> |{logstash-ref}/configuration-file-structure.html#string[string], (`beginning`, `end`, or `look_back`)|No +| <<{version}-plugins-{type}s-{plugin}-initial_position_look_back>> |{logstash-ref}/configuration-file-structure.html#number[number]|No, unless `initial_position => look_back` +| <<{version}-plugins-{type}s-{plugin}-max_batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-storage_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_container>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +NOTE: All Event Hubs options are common to both basic and advanced +configurations, with the following exceptions. The basic configuration uses +`event_hub_connections` to support multiple connections. The advanced +configuration uses `event_hubs` and `event_hub_connection` (singular). + +[id="{version}-plugins-{type}s-{plugin}-config_mode"] +===== `config_mode` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid entries are `basic` or `advanced` +* Default value is `basic` + +Sets configuration to either <<{version}-plugins-{type}s-{plugin}-eh_basic_config>> or <<{version}-plugins-{type}s-{plugin}-eh_advanced_config>>. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hubs"] +===== `event_hubs` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Ignored for basic configuration +* Required for advanced configuration + +Defines the Event Hubs to be read. An array of hashes where each entry is a +hash of the Event Hub name and its configuration options. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1..." + }}, + { "event_hub_name2" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" + }} + ] + consumer_group => "logstash" # shared across all Event Hubs +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connections"] +===== `event_hub_connections` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Required for basic configuration + +List of connection strings that identifies the Event Hubs to be read. Connection +strings include the EntityPath for the Event Hub. + +The `event_hub_connections` option is defined +per Event Hub. All other configuration options are shared among Event Hubs. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connection"] +===== `event_hub_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value +* Valid only for advanced configuration + +Connection string that identifies the Event Hub to be read. Advanced +configuration options can be set per Event Hub. This option modifies +`event_hub_name`, and should be nested under it. (See sample.) This option +accepts only one connection string. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1...;EntityPath=event_hub_name1" + }} + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-checkpoint_interval"] +===== `checkpoint_interval` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `5` seconds +* Set to `0` to disable. + +Interval in seconds to write checkpoints during batch processing. Checkpoints +tell Logstash where to resume processing after a restart. Checkpoints are +automatically written at the end of each batch, regardless of this setting. + +Writing checkpoints too frequently can slow down processing unnecessarily. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + checkpoint_interval => 5 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-consumer_group"] +===== `consumer_group` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `$Default` + +Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that +consumer group so that they can work together properly. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + consumer_group => "logstash" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Adds metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + decorate_events => true +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position"] +===== `initial_position` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid arguments are `beginning`, `end`, `look_back` +* Default value is `beginning` + +When first reading from an Event Hub, start from this position: + +* `beginning` reads all pre-existing events in the Event Hub +* `end` does not read any pre-existing events in the Event Hub +* `look_back` reads `end` minus a number of seconds worth of pre-existing events. +You control the number of seconds using the `initial_position_look_back` option. + +Note: If `storage_connection` is set, the `initial_position` value is used only +the first time Logstash reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "beginning" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position_look_back"] +===== `initial_position_look_back` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `86400` +* Used only if `initial_position` is set to `look-back` + +Number of seconds to look back to find the initial position for pre-existing +events. This option is used only if `initial_position` is set to `look_back`. If +`storage_connection` is set, this configuration applies only the first time Logstash +reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "look_back" + initial_position_look_back => 86400 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-max_batch_size"] +===== `max_batch_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `125` + +Maximum number of events retrieved and processed together. A checkpoint is +created after each batch. Increasing this value may help with performance, but +requires more memory. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + max_batch_size => 125 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_connection"] +===== `storage_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value + +Connection string for blob account storage. Blob account storage persists the +offsets between restarts, and ensures that multiple instances of Logstash +process different partitions. +When this value is set, restarts resume where processing left off. +When this value is not set, the `initial_position` value is used on every restart. + +We strongly recommend that you define this value for production environments. + + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_container"] +===== `storage_container` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Defaults to the Event Hub name if not defined + +Name of the storage container used to persist offsets and allow multiple instances of Logstash +to work together. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" +} +---- + +To avoid overwriting offsets, you can use different storage containers. This is +particularly important if you are monitoring two Event Hubs with the same name. +You can use the advanced configuration model to configure different storage +containers. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + storage_container => "insights-operational-logs-1" + }}, + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_container => "insights-operational-logs-2" + }} + ] + } +---- + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Minimum value is `2` +* Default value is `16` + +Total number of threads used to process events. The value you set here applies +to all Event Hubs. Even with advanced configuration, this value is a global +setting, and can't be set per event hub. + +[source,ruby] +---- +azure_event_hubs { + threads => 16 +} +---- + +The number of threads should be the number of Event Hubs plus one or more. +See <<{version}-plugins-{type}s-{plugin}-best-practices>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.9.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.9.asciidoc new file mode 100644 index 000000000..7c6ca1b88 --- /dev/null +++ b/docs/versioned-plugins/inputs/azure_event_hubs-v1.4.9.asciidoc @@ -0,0 +1,536 @@ +:plugin: azure_event_hubs +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.4.9 +:release_date: 2024-10-02 +:changelog_url: https://github.com/logstash-plugins/logstash-input-azure_event_hubs/blob/v1.4.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Azure Event Hubs plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This plugin consumes events from +https://azure.microsoft.com/en-us/services/event-hubs[Azure Event Hubs], a +highly scalable data streaming platform and event ingestion service. Event +producers send events to the Azure Event Hub, and this plugin consumes those +events for use with Logstash. + +Many Azure services integrate with the Azure Event Hubs. +https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Azure +Monitor], for example, integrates with Azure Event Hubs to provide infrastructure metrics. + +IMPORTANT: This plugin requires outbound connections to ports `tcp/443`, `tcp/9093`, `tcp/5671`, and `tcp/5672`, +as noted in the https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall[Microsoft Event Hub documentation]. + +===== Event Hub connection string + +The plugin uses the connection string to access Azure Events Hubs. Find the +connection string here: https://portal.azure.com[Azure Portal]`-> Event Hub -> +Shared access polices`. The event_hub_connections option passes the Event Hub +connection strings for the basic configuration. + +Sample connection string: + +[source,text] +---- +Endpoint=sb://logstash.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs +---- + +===== Blob Storage and connection string + +https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage +account] is an essential part of Azure-to-Logstash configuration. +A Blob Storage account is a central location that enables multiple instances of +Logstash to work together to process events. It records the +offset (location) of processed events. On restart, Logstash resumes processing +exactly where it left off. + +Configuration notes: + +* A Blob Storage account is highly recommended for use with this plugin, and is +likely required for production servers. +* The `storage_connection` option passes the blob storage connection string. +* Configure all Logstash instances to use the same `storage_connection` to get the +benefits of shared processing. + +Sample Blob Storage connection string: + +[source,text] +---- +DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net +---- + +Find the connection string to Blob Storage here: +https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. + +[id="{version}-plugins-{type}s-{plugin}-best-practices"] +===== Best practices + +Here are some guidelines to help you avoid data conflicts that can cause lost +events. + +* <<{version}-plugins-{type}s-{plugin}-bp-group>> +* <<{version}-plugins-{type}s-{plugin}-bp-multihub>> +* <<{version}-plugins-{type}s-{plugin}-bp-threads>> + +[id="{version}-plugins-{type}s-{plugin}-bp-group"] +===== Create a Logstash consumer group +Create a new consumer group specifically for Logstash. Do not use the $default or +any other consumer group that might already be in use. Reusing consumer groups +among non-related consumers can cause unexpected behavior and possibly lost +events. All Logstash instances should use the same consumer group so that they can +work together for processing events. + +[id="{version}-plugins-{type}s-{plugin}-bp-multihub"] +===== Avoid overwriting offset with multiple Event Hubs +The offsets (position) of the Event Hubs are stored in the configured Azure Blob +store. The Azure Blob store uses paths like a file system to store the offsets. +If the paths between multiple Event Hubs overlap, then the offsets may be stored +incorrectly. + +To avoid duplicate file paths, use the advanced configuration model and make +sure that at least one of these options is different per Event Hub: + +** storage_connection +** storage_container (defaults to Event Hub name if not defined) +** consumer_group + + +[id="{version}-plugins-{type}s-{plugin}-bp-threads"] +===== Set number of threads correctly + +By default, the number of threads used to service all event hubs is `16`. And while this +may be sufficient for most use cases, throughput may be improved by refining this number. +When servicing a large number of partitions across one or more event hubs, setting a higher +value may result in improved performance. The maximum number of threads is not strictly bound +by the total number of partitions being serviced, but setting the value much higher than +that may mean that some threads are idle. + +NOTE: The number of threads *must* be greater than or equal to the number of Event hubs plus one. + +NOTE: Threads are currently available only as a global setting across all event hubs in a single `azure_event_hubs` +input definition. However if your configuration includes multiple `azure_event_hubs` inputs, the threads setting applies +independently to each. + +**Example: Single event hub** + +If you’re collecting activity logs from one event hub instance, +then only 2 threads are required. + +* Event hubs = 1 +* Minimum threads = 2 (1 Event Hub + 1) + +**Example: Multiple event hubs** + +If you are collecting activity logs from more than event hub instance, then at least 1 thread per event hub is required. + +* Event hubs = 4 +* Minimum threads = 5 (4 Event Hubs + 1) + +When you are using multiple partitions per event hub, you may want to assign more threads. +A good base level is (1 + `number of event hubs * number of partitions`). +That is, one thread for each partition across all event hubs. + +[id="{version}-plugins-{type}s-{plugin}-eh_config_models"] +==== Configuration models + +This plugin supports two configuration models: basic and advanced. Basic +configuration is recommended for most use cases, and is illustrated in the +examples throughout this topic. + +[id="{version}-plugins-{type}s-{plugin}-eh_basic_config"] +===== Basic configuration (default) + +Basic configuration is the default and supports consuming from multiple Event +Hubs. All Events Hubs, except for the connection string, share the same +configuration. + +You supply a list of Event Hub connection strings, complete with the Event Hub +EntityPath that defines the Event Hub name. All other configuration settings are +shared. + +[source,ruby] +---- +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...EntityPath=insights-logs-errors", "Endpoint=sb://example2...EntityPath=insights-metrics-pt1m"] + threads => 8 + decorate_events => true + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-eh_advanced_config"] +===== Advanced configuration +The advanced configuration model accommodates deployments where different Event +Hubs require different configurations. Options can be configured per Event Hub. +You provide a list of Event Hub names through the `event_hubs` option. Under +each name, specify the configuration for that Event Hub. Options can be defined +globally or expressed per Event Hub. + +If the same configuration option appears in both the global and `event_hub` +sections, the more specific (event_hub) setting takes precedence. + +NOTE: Advanced configuration is not necessary or recommended for most use cases. + +[source,ruby] +---- +input { + azure_event_hubs { + config_mode => "advanced" + threads => 8 + decorate_events => true + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + initial_position => "beginning" + consumer_group => "iam_team" + }}, + {"insights-metrics-pt1m" => { + event_hub_connection => "Endpoint=sb://example2..." + initial_position => "end" + consumer_group => "db_team" + }} + ] + } +} +---- + +In this example, `storage_connection` and `decorate_events` are applied globally. +The two Event Hubs each have their own settings for `consumer_groups` +and `initial_position`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Azure Event Hubs Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-config_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], (`basic` or `advanced`)|No +| <<{version}-plugins-{type}s-{plugin}-event_hubs>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connections>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => basic` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-checkpoint_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_group>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-initial_position>> |{logstash-ref}/configuration-file-structure.html#string[string], (`beginning`, `end`, or `look_back`)|No +| <<{version}-plugins-{type}s-{plugin}-initial_position_look_back>> |{logstash-ref}/configuration-file-structure.html#number[number]|No, unless `initial_position => look_back` +| <<{version}-plugins-{type}s-{plugin}-max_batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-storage_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_container>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +NOTE: All Event Hubs options are common to both basic and advanced +configurations, with the following exceptions. The basic configuration uses +`event_hub_connections` to support multiple connections. The advanced +configuration uses `event_hubs` and `event_hub_connection` (singular). + +[id="{version}-plugins-{type}s-{plugin}-config_mode"] +===== `config_mode` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid entries are `basic` or `advanced` +* Default value is `basic` + +Sets configuration to either <<{version}-plugins-{type}s-{plugin}-eh_basic_config>> or <<{version}-plugins-{type}s-{plugin}-eh_advanced_config>>. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hubs"] +===== `event_hubs` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Ignored for basic configuration +* Required for advanced configuration + +Defines the Event Hubs to be read. An array of hashes where each entry is a +hash of the Event Hub name and its configuration options. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1..." + }}, + { "event_hub_name2" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" + }} + ] + consumer_group => "logstash" # shared across all Event Hubs +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connections"] +===== `event_hub_connections` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Required for basic configuration + +List of connection strings that identifies the Event Hubs to be read. Connection +strings include the EntityPath for the Event Hub. + +The `event_hub_connections` option is defined +per Event Hub. All other configuration options are shared among Event Hubs. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connection"] +===== `event_hub_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value +* Valid only for advanced configuration + +Connection string that identifies the Event Hub to be read. Advanced +configuration options can be set per Event Hub. This option modifies +`event_hub_name`, and should be nested under it. (See sample.) This option +accepts only one connection string. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1...;EntityPath=event_hub_name1" + }} + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-checkpoint_interval"] +===== `checkpoint_interval` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `5` seconds +* Set to `0` to disable. + +Interval in seconds to write checkpoints during batch processing. Checkpoints +tell Logstash where to resume processing after a restart. Checkpoints are +automatically written at the end of each batch, regardless of this setting. + +Writing checkpoints too frequently can slow down processing unnecessarily. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + checkpoint_interval => 5 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-consumer_group"] +===== `consumer_group` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `$Default` + +Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that +consumer group so that they can work together properly. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + consumer_group => "logstash" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Adds metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + decorate_events => true +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position"] +===== `initial_position` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid arguments are `beginning`, `end`, `look_back` +* Default value is `beginning` + +When first reading from an Event Hub, start from this position: + +* `beginning` reads all pre-existing events in the Event Hub +* `end` does not read any pre-existing events in the Event Hub +* `look_back` reads `end` minus a number of seconds worth of pre-existing events. +You control the number of seconds using the `initial_position_look_back` option. + +Note: If `storage_connection` is set, the `initial_position` value is used only +the first time Logstash reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "beginning" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position_look_back"] +===== `initial_position_look_back` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `86400` +* Used only if `initial_position` is set to `look-back` + +Number of seconds to look back to find the initial position for pre-existing +events. This option is used only if `initial_position` is set to `look_back`. If +`storage_connection` is set, this configuration applies only the first time Logstash +reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "look_back" + initial_position_look_back => 86400 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-max_batch_size"] +===== `max_batch_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `125` + +Maximum number of events retrieved and processed together. A checkpoint is +created after each batch. Increasing this value may help with performance, but +requires more memory. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + max_batch_size => 125 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_connection"] +===== `storage_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value + +Connection string for blob account storage. Blob account storage persists the +offsets between restarts, and ensures that multiple instances of Logstash +process different partitions. +When this value is set, restarts resume where processing left off. +When this value is not set, the `initial_position` value is used on every restart. + +We strongly recommend that you define this value for production environments. + + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_container"] +===== `storage_container` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Defaults to the Event Hub name if not defined + +Name of the storage container used to persist offsets and allow multiple instances of Logstash +to work together. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" +} +---- + +To avoid overwriting offsets, you can use different storage containers. This is +particularly important if you are monitoring two Event Hubs with the same name. +You can use the advanced configuration model to configure different storage +containers. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + storage_container => "insights-operational-logs-1" + }}, + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_container => "insights-operational-logs-2" + }} + ] + } +---- + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Minimum value is `2` +* Default value is `16` + +Total number of threads used to process events. The value you set here applies +to all Event Hubs. Even with advanced configuration, this value is a global +setting, and can't be set per event hub. + +[source,ruby] +---- +azure_event_hubs { + threads => 16 +} +---- + +The number of threads should be the number of Event Hubs plus one or more. +See <<{version}-plugins-{type}s-{plugin}-best-practices>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.0.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.0.asciidoc new file mode 100644 index 000000000..885a3427b --- /dev/null +++ b/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.0.asciidoc @@ -0,0 +1,536 @@ +:plugin: azure_event_hubs +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.5.0 +:release_date: 2024-10-25 +:changelog_url: https://github.com/logstash-plugins/logstash-input-azure_event_hubs/blob/v1.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Azure Event Hubs plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This plugin consumes events from +https://azure.microsoft.com/en-us/services/event-hubs[Azure Event Hubs], a +highly scalable data streaming platform and event ingestion service. Event +producers send events to the Azure Event Hub, and this plugin consumes those +events for use with Logstash. + +Many Azure services integrate with the Azure Event Hubs. +https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Azure +Monitor], for example, integrates with Azure Event Hubs to provide infrastructure metrics. + +IMPORTANT: This plugin requires outbound connections to ports `tcp/443`, `tcp/9093`, `tcp/5671`, and `tcp/5672`, +as noted in the https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall[Microsoft Event Hub documentation]. + +===== Event Hub connection string + +The plugin uses the connection string to access Azure Events Hubs. Find the +connection string here: https://portal.azure.com[Azure Portal]`-> Event Hub -> +Shared access polices`. The event_hub_connections option passes the Event Hub +connection strings for the basic configuration. + +Sample connection string: + +[source,text] +---- +Endpoint=sb://logstash.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs +---- + +===== Blob Storage and connection string + +https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage +account] is an essential part of Azure-to-Logstash configuration. +A Blob Storage account is a central location that enables multiple instances of +Logstash to work together to process events. It records the +offset (location) of processed events. On restart, Logstash resumes processing +exactly where it left off. + +Configuration notes: + +* A Blob Storage account is highly recommended for use with this plugin, and is +likely required for production servers. +* The `storage_connection` option passes the blob storage connection string. +* Configure all Logstash instances to use the same `storage_connection` to get the +benefits of shared processing. + +Sample Blob Storage connection string: + +[source,text] +---- +DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net +---- + +Find the connection string to Blob Storage here: +https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. + +[id="{version}-plugins-{type}s-{plugin}-best-practices"] +===== Best practices + +Here are some guidelines to help you avoid data conflicts that can cause lost +events. + +* <<{version}-plugins-{type}s-{plugin}-bp-group>> +* <<{version}-plugins-{type}s-{plugin}-bp-multihub>> +* <<{version}-plugins-{type}s-{plugin}-bp-threads>> + +[id="{version}-plugins-{type}s-{plugin}-bp-group"] +===== Create a Logstash consumer group +Create a new consumer group specifically for Logstash. Do not use the $default or +any other consumer group that might already be in use. Reusing consumer groups +among non-related consumers can cause unexpected behavior and possibly lost +events. All Logstash instances should use the same consumer group so that they can +work together for processing events. + +[id="{version}-plugins-{type}s-{plugin}-bp-multihub"] +===== Avoid overwriting offset with multiple Event Hubs +The offsets (position) of the Event Hubs are stored in the configured Azure Blob +store. The Azure Blob store uses paths like a file system to store the offsets. +If the paths between multiple Event Hubs overlap, then the offsets may be stored +incorrectly. + +To avoid duplicate file paths, use the advanced configuration model and make +sure that at least one of these options is different per Event Hub: + +** storage_connection +** storage_container (defaults to Event Hub name if not defined) +** consumer_group + + +[id="{version}-plugins-{type}s-{plugin}-bp-threads"] +===== Set number of threads correctly + +By default, the number of threads used to service all event hubs is `16`. And while this +may be sufficient for most use cases, throughput may be improved by refining this number. +When servicing a large number of partitions across one or more event hubs, setting a higher +value may result in improved performance. The maximum number of threads is not strictly bound +by the total number of partitions being serviced, but setting the value much higher than +that may mean that some threads are idle. + +NOTE: The number of threads *must* be greater than or equal to the number of Event hubs plus one. + +NOTE: Threads are currently available only as a global setting across all event hubs in a single `azure_event_hubs` +input definition. However if your configuration includes multiple `azure_event_hubs` inputs, the threads setting applies +independently to each. + +**Example: Single event hub** + +If you’re collecting activity logs from one event hub instance, +then only 2 threads are required. + +* Event hubs = 1 +* Minimum threads = 2 (1 Event Hub + 1) + +**Example: Multiple event hubs** + +If you are collecting activity logs from more than event hub instance, then at least 1 thread per event hub is required. + +* Event hubs = 4 +* Minimum threads = 5 (4 Event Hubs + 1) + +When you are using multiple partitions per event hub, you may want to assign more threads. +A good base level is (1 + `number of event hubs * number of partitions`). +That is, one thread for each partition across all event hubs. + +[id="{version}-plugins-{type}s-{plugin}-eh_config_models"] +==== Configuration models + +This plugin supports two configuration models: basic and advanced. Basic +configuration is recommended for most use cases, and is illustrated in the +examples throughout this topic. + +[id="{version}-plugins-{type}s-{plugin}-eh_basic_config"] +===== Basic configuration (default) + +Basic configuration is the default and supports consuming from multiple Event +Hubs. All Events Hubs, except for the connection string, share the same +configuration. + +You supply a list of Event Hub connection strings, complete with the Event Hub +EntityPath that defines the Event Hub name. All other configuration settings are +shared. + +[source,ruby] +---- +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...EntityPath=insights-logs-errors", "Endpoint=sb://example2...EntityPath=insights-metrics-pt1m"] + threads => 8 + decorate_events => true + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-eh_advanced_config"] +===== Advanced configuration +The advanced configuration model accommodates deployments where different Event +Hubs require different configurations. Options can be configured per Event Hub. +You provide a list of Event Hub names through the `event_hubs` option. Under +each name, specify the configuration for that Event Hub. Options can be defined +globally or expressed per Event Hub. + +If the same configuration option appears in both the global and `event_hub` +sections, the more specific (event_hub) setting takes precedence. + +NOTE: Advanced configuration is not necessary or recommended for most use cases. + +[source,ruby] +---- +input { + azure_event_hubs { + config_mode => "advanced" + threads => 8 + decorate_events => true + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + initial_position => "beginning" + consumer_group => "iam_team" + }}, + {"insights-metrics-pt1m" => { + event_hub_connection => "Endpoint=sb://example2..." + initial_position => "end" + consumer_group => "db_team" + }} + ] + } +} +---- + +In this example, `storage_connection` and `decorate_events` are applied globally. +The two Event Hubs each have their own settings for `consumer_groups` +and `initial_position`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Azure Event Hubs Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-config_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], (`basic` or `advanced`)|No +| <<{version}-plugins-{type}s-{plugin}-event_hubs>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connections>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => basic` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-checkpoint_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_group>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-initial_position>> |{logstash-ref}/configuration-file-structure.html#string[string], (`beginning`, `end`, or `look_back`)|No +| <<{version}-plugins-{type}s-{plugin}-initial_position_look_back>> |{logstash-ref}/configuration-file-structure.html#number[number]|No, unless `initial_position => look_back` +| <<{version}-plugins-{type}s-{plugin}-max_batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-storage_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_container>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +NOTE: All Event Hubs options are common to both basic and advanced +configurations, with the following exceptions. The basic configuration uses +`event_hub_connections` to support multiple connections. The advanced +configuration uses `event_hubs` and `event_hub_connection` (singular). + +[id="{version}-plugins-{type}s-{plugin}-config_mode"] +===== `config_mode` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid entries are `basic` or `advanced` +* Default value is `basic` + +Sets configuration to either <<{version}-plugins-{type}s-{plugin}-eh_basic_config>> or <<{version}-plugins-{type}s-{plugin}-eh_advanced_config>>. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hubs"] +===== `event_hubs` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Ignored for basic configuration +* Required for advanced configuration + +Defines the Event Hubs to be read. An array of hashes where each entry is a +hash of the Event Hub name and its configuration options. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1..." + }}, + { "event_hub_name2" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" + }} + ] + consumer_group => "logstash" # shared across all Event Hubs +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connections"] +===== `event_hub_connections` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Required for basic configuration + +List of connection strings that identifies the Event Hubs to be read. Connection +strings include the EntityPath for the Event Hub. + +The `event_hub_connections` option is defined +per Event Hub. All other configuration options are shared among Event Hubs. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connection"] +===== `event_hub_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value +* Valid only for advanced configuration + +Connection string that identifies the Event Hub to be read. Advanced +configuration options can be set per Event Hub. This option modifies +`event_hub_name`, and should be nested under it. (See sample.) This option +accepts only one connection string. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1...;EntityPath=event_hub_name1" + }} + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-checkpoint_interval"] +===== `checkpoint_interval` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `5` seconds +* Set to `0` to disable. + +Interval in seconds to write checkpoints during batch processing. Checkpoints +tell Logstash where to resume processing after a restart. Checkpoints are +automatically written at the end of each batch, regardless of this setting. + +Writing checkpoints too frequently can slow down processing unnecessarily. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + checkpoint_interval => 5 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-consumer_group"] +===== `consumer_group` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `$Default` + +Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that +consumer group so that they can work together properly. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + consumer_group => "logstash" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Adds metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + decorate_events => true +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position"] +===== `initial_position` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid arguments are `beginning`, `end`, `look_back` +* Default value is `beginning` + +When first reading from an Event Hub, start from this position: + +* `beginning` reads all pre-existing events in the Event Hub +* `end` does not read any pre-existing events in the Event Hub +* `look_back` reads `end` minus a number of seconds worth of pre-existing events. +You control the number of seconds using the `initial_position_look_back` option. + +Note: If `storage_connection` is set, the `initial_position` value is used only +the first time Logstash reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "beginning" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position_look_back"] +===== `initial_position_look_back` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `86400` +* Used only if `initial_position` is set to `look-back` + +Number of seconds to look back to find the initial position for pre-existing +events. This option is used only if `initial_position` is set to `look_back`. If +`storage_connection` is set, this configuration applies only the first time Logstash +reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "look_back" + initial_position_look_back => 86400 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-max_batch_size"] +===== `max_batch_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `125` + +Maximum number of events retrieved and processed together. A checkpoint is +created after each batch. Increasing this value may help with performance, but +requires more memory. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + max_batch_size => 125 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_connection"] +===== `storage_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value + +Connection string for blob account storage. Blob account storage persists the +offsets between restarts, and ensures that multiple instances of Logstash +process different partitions. +When this value is set, restarts resume where processing left off. +When this value is not set, the `initial_position` value is used on every restart. + +We strongly recommend that you define this value for production environments. + + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_container"] +===== `storage_container` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Defaults to the Event Hub name if not defined + +Name of the storage container used to persist offsets and allow multiple instances of Logstash +to work together. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" +} +---- + +To avoid overwriting offsets, you can use different storage containers. This is +particularly important if you are monitoring two Event Hubs with the same name. +You can use the advanced configuration model to configure different storage +containers. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + storage_container => "insights-operational-logs-1" + }}, + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_container => "insights-operational-logs-2" + }} + ] + } +---- + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Minimum value is `2` +* Default value is `16` + +Total number of threads used to process events. The value you set here applies +to all Event Hubs. Even with advanced configuration, this value is a global +setting, and can't be set per event hub. + +[source,ruby] +---- +azure_event_hubs { + threads => 16 +} +---- + +The number of threads should be the number of Event Hubs plus one or more. +See <<{version}-plugins-{type}s-{plugin}-best-practices>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.1.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.1.asciidoc new file mode 100644 index 000000000..a0d7024d4 --- /dev/null +++ b/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.1.asciidoc @@ -0,0 +1,536 @@ +:plugin: azure_event_hubs +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.5.1 +:release_date: 2025-01-03 +:changelog_url: https://github.com/logstash-plugins/logstash-input-azure_event_hubs/blob/v1.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Azure Event Hubs plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This plugin consumes events from +https://azure.microsoft.com/en-us/services/event-hubs[Azure Event Hubs], a +highly scalable data streaming platform and event ingestion service. Event +producers send events to the Azure Event Hub, and this plugin consumes those +events for use with Logstash. + +Many Azure services integrate with the Azure Event Hubs. +https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Azure +Monitor], for example, integrates with Azure Event Hubs to provide infrastructure metrics. + +IMPORTANT: This plugin requires outbound connections to ports `tcp/443`, `tcp/9093`, `tcp/5671`, and `tcp/5672`, +as noted in the https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall[Microsoft Event Hub documentation]. + +===== Event Hub connection string + +The plugin uses the connection string to access Azure Events Hubs. Find the +connection string here: https://portal.azure.com[Azure Portal]`-> Event Hub -> +Shared access polices`. The event_hub_connections option passes the Event Hub +connection strings for the basic configuration. + +Sample connection string: + +[source,text] +---- +Endpoint=sb://logstash.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs +---- + +===== Blob Storage and connection string + +https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage +account] is an essential part of Azure-to-Logstash configuration. +A Blob Storage account is a central location that enables multiple instances of +Logstash to work together to process events. It records the +offset (location) of processed events. On restart, Logstash resumes processing +exactly where it left off. + +Configuration notes: + +* A Blob Storage account is highly recommended for use with this plugin, and is +likely required for production servers. +* The `storage_connection` option passes the blob storage connection string. +* Configure all Logstash instances to use the same `storage_connection` to get the +benefits of shared processing. + +Sample Blob Storage connection string: + +[source,text] +---- +DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net +---- + +Find the connection string to Blob Storage here: +https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. + +[id="{version}-plugins-{type}s-{plugin}-best-practices"] +===== Best practices + +Here are some guidelines to help you avoid data conflicts that can cause lost +events. + +* <<{version}-plugins-{type}s-{plugin}-bp-group>> +* <<{version}-plugins-{type}s-{plugin}-bp-multihub>> +* <<{version}-plugins-{type}s-{plugin}-bp-threads>> + +[id="{version}-plugins-{type}s-{plugin}-bp-group"] +===== Create a Logstash consumer group +Create a new consumer group specifically for Logstash. Do not use the $default or +any other consumer group that might already be in use. Reusing consumer groups +among non-related consumers can cause unexpected behavior and possibly lost +events. All Logstash instances should use the same consumer group so that they can +work together for processing events. + +[id="{version}-plugins-{type}s-{plugin}-bp-multihub"] +===== Avoid overwriting offset with multiple Event Hubs +The offsets (position) of the Event Hubs are stored in the configured Azure Blob +store. The Azure Blob store uses paths like a file system to store the offsets. +If the paths between multiple Event Hubs overlap, then the offsets may be stored +incorrectly. + +To avoid duplicate file paths, use the advanced configuration model and make +sure that at least one of these options is different per Event Hub: + +** storage_connection +** storage_container (defaults to Event Hub name if not defined) +** consumer_group + + +[id="{version}-plugins-{type}s-{plugin}-bp-threads"] +===== Set number of threads correctly + +By default, the number of threads used to service all event hubs is `16`. And while this +may be sufficient for most use cases, throughput may be improved by refining this number. +When servicing a large number of partitions across one or more event hubs, setting a higher +value may result in improved performance. The maximum number of threads is not strictly bound +by the total number of partitions being serviced, but setting the value much higher than +that may mean that some threads are idle. + +NOTE: The number of threads *must* be greater than or equal to the number of Event hubs plus one. + +NOTE: Threads are currently available only as a global setting across all event hubs in a single `azure_event_hubs` +input definition. However if your configuration includes multiple `azure_event_hubs` inputs, the threads setting applies +independently to each. + +**Example: Single event hub** + +If you’re collecting activity logs from one event hub instance, +then only 2 threads are required. + +* Event hubs = 1 +* Minimum threads = 2 (1 Event Hub + 1) + +**Example: Multiple event hubs** + +If you are collecting activity logs from more than event hub instance, then at least 1 thread per event hub is required. + +* Event hubs = 4 +* Minimum threads = 5 (4 Event Hubs + 1) + +When you are using multiple partitions per event hub, you may want to assign more threads. +A good base level is (1 + `number of event hubs * number of partitions`). +That is, one thread for each partition across all event hubs. + +[id="{version}-plugins-{type}s-{plugin}-eh_config_models"] +==== Configuration models + +This plugin supports two configuration models: basic and advanced. Basic +configuration is recommended for most use cases, and is illustrated in the +examples throughout this topic. + +[id="{version}-plugins-{type}s-{plugin}-eh_basic_config"] +===== Basic configuration (default) + +Basic configuration is the default and supports consuming from multiple Event +Hubs. All Events Hubs, except for the connection string, share the same +configuration. + +You supply a list of Event Hub connection strings, complete with the Event Hub +EntityPath that defines the Event Hub name. All other configuration settings are +shared. + +[source,ruby] +---- +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...EntityPath=insights-logs-errors", "Endpoint=sb://example2...EntityPath=insights-metrics-pt1m"] + threads => 8 + decorate_events => true + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-eh_advanced_config"] +===== Advanced configuration +The advanced configuration model accommodates deployments where different Event +Hubs require different configurations. Options can be configured per Event Hub. +You provide a list of Event Hub names through the `event_hubs` option. Under +each name, specify the configuration for that Event Hub. Options can be defined +globally or expressed per Event Hub. + +If the same configuration option appears in both the global and `event_hub` +sections, the more specific (event_hub) setting takes precedence. + +NOTE: Advanced configuration is not necessary or recommended for most use cases. + +[source,ruby] +---- +input { + azure_event_hubs { + config_mode => "advanced" + threads => 8 + decorate_events => true + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + initial_position => "beginning" + consumer_group => "iam_team" + }}, + {"insights-metrics-pt1m" => { + event_hub_connection => "Endpoint=sb://example2..." + initial_position => "end" + consumer_group => "db_team" + }} + ] + } +} +---- + +In this example, `storage_connection` and `decorate_events` are applied globally. +The two Event Hubs each have their own settings for `consumer_groups` +and `initial_position`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Azure Event Hubs Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-config_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], (`basic` or `advanced`)|No +| <<{version}-plugins-{type}s-{plugin}-event_hubs>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connections>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => basic` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-checkpoint_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_group>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-initial_position>> |{logstash-ref}/configuration-file-structure.html#string[string], (`beginning`, `end`, or `look_back`)|No +| <<{version}-plugins-{type}s-{plugin}-initial_position_look_back>> |{logstash-ref}/configuration-file-structure.html#number[number]|No, unless `initial_position => look_back` +| <<{version}-plugins-{type}s-{plugin}-max_batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-storage_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_container>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +NOTE: All Event Hubs options are common to both basic and advanced +configurations, with the following exceptions. The basic configuration uses +`event_hub_connections` to support multiple connections. The advanced +configuration uses `event_hubs` and `event_hub_connection` (singular). + +[id="{version}-plugins-{type}s-{plugin}-config_mode"] +===== `config_mode` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid entries are `basic` or `advanced` +* Default value is `basic` + +Sets configuration to either <<{version}-plugins-{type}s-{plugin}-eh_basic_config>> or <<{version}-plugins-{type}s-{plugin}-eh_advanced_config>>. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hubs"] +===== `event_hubs` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Ignored for basic configuration +* Required for advanced configuration + +Defines the Event Hubs to be read. An array of hashes where each entry is a +hash of the Event Hub name and its configuration options. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1..." + }}, + { "event_hub_name2" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" + }} + ] + consumer_group => "logstash" # shared across all Event Hubs +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connections"] +===== `event_hub_connections` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Required for basic configuration + +List of connection strings that identifies the Event Hubs to be read. Connection +strings include the EntityPath for the Event Hub. + +The `event_hub_connections` option is defined +per Event Hub. All other configuration options are shared among Event Hubs. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connection"] +===== `event_hub_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value +* Valid only for advanced configuration + +Connection string that identifies the Event Hub to be read. Advanced +configuration options can be set per Event Hub. This option modifies +`event_hub_name`, and should be nested under it. (See sample.) This option +accepts only one connection string. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1...;EntityPath=event_hub_name1" + }} + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-checkpoint_interval"] +===== `checkpoint_interval` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `5` seconds +* Set to `0` to disable. + +Interval in seconds to write checkpoints during batch processing. Checkpoints +tell Logstash where to resume processing after a restart. Checkpoints are +automatically written at the end of each batch, regardless of this setting. + +Writing checkpoints too frequently can slow down processing unnecessarily. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + checkpoint_interval => 5 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-consumer_group"] +===== `consumer_group` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `$Default` + +Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that +consumer group so that they can work together properly. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + consumer_group => "logstash" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Adds metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + decorate_events => true +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position"] +===== `initial_position` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid arguments are `beginning`, `end`, `look_back` +* Default value is `beginning` + +When first reading from an Event Hub, start from this position: + +* `beginning` reads all pre-existing events in the Event Hub +* `end` does not read any pre-existing events in the Event Hub +* `look_back` reads `end` minus a number of seconds worth of pre-existing events. +You control the number of seconds using the `initial_position_look_back` option. + +Note: If `storage_connection` is set, the `initial_position` value is used only +the first time Logstash reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "beginning" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position_look_back"] +===== `initial_position_look_back` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `86400` +* Used only if `initial_position` is set to `look-back` + +Number of seconds to look back to find the initial position for pre-existing +events. This option is used only if `initial_position` is set to `look_back`. If +`storage_connection` is set, this configuration applies only the first time Logstash +reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "look_back" + initial_position_look_back => 86400 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-max_batch_size"] +===== `max_batch_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `125` + +Maximum number of events retrieved and processed together. A checkpoint is +created after each batch. Increasing this value may help with performance, but +requires more memory. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + max_batch_size => 125 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_connection"] +===== `storage_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value + +Connection string for blob account storage. Blob account storage persists the +offsets between restarts, and ensures that multiple instances of Logstash +process different partitions. +When this value is set, restarts resume where processing left off. +When this value is not set, the `initial_position` value is used on every restart. + +We strongly recommend that you define this value for production environments. + + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_container"] +===== `storage_container` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Defaults to the Event Hub name if not defined + +Name of the storage container used to persist offsets and allow multiple instances of Logstash +to work together. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" +} +---- + +To avoid overwriting offsets, you can use different storage containers. This is +particularly important if you are monitoring two Event Hubs with the same name. +You can use the advanced configuration model to configure different storage +containers. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + storage_container => "insights-operational-logs-1" + }}, + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_container => "insights-operational-logs-2" + }} + ] + } +---- + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Minimum value is `2` +* Default value is `16` + +Total number of threads used to process events. The value you set here applies +to all Event Hubs. Even with advanced configuration, this value is a global +setting, and can't be set per event hub. + +[source,ruby] +---- +azure_event_hubs { + threads => 16 +} +---- + +The number of threads should be the number of Event Hubs plus one or more. +See <<{version}-plugins-{type}s-{plugin}-best-practices>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.2.asciidoc b/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.2.asciidoc new file mode 100644 index 000000000..d74074901 --- /dev/null +++ b/docs/versioned-plugins/inputs/azure_event_hubs-v1.5.2.asciidoc @@ -0,0 +1,536 @@ +:plugin: azure_event_hubs +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.5.2 +:release_date: 2025-07-11 +:changelog_url: https://github.com/logstash-plugins/logstash-input-azure_event_hubs/blob/v1.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Azure Event Hubs plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This plugin consumes events from +https://azure.microsoft.com/en-us/services/event-hubs[Azure Event Hubs], a +highly scalable data streaming platform and event ingestion service. Event +producers send events to the Azure Event Hub, and this plugin consumes those +events for use with Logstash. + +Many Azure services integrate with the Azure Event Hubs. +https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Azure +Monitor], for example, integrates with Azure Event Hubs to provide infrastructure metrics. + +IMPORTANT: This plugin requires outbound connections to ports `tcp/443`, `tcp/9093`, `tcp/5671`, and `tcp/5672`, +as noted in the https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall[Microsoft Event Hub documentation]. + +===== Event Hub connection string + +The plugin uses the connection string to access Azure Events Hubs. Find the +connection string here: https://portal.azure.com[Azure Portal]`-> Event Hub -> +Shared access polices`. The event_hub_connections option passes the Event Hub +connection strings for the basic configuration. + +Sample connection string: + +[source,text] +---- +Endpoint=sb://logstash.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs +---- + +===== Blob Storage and connection string + +https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage +account] is an essential part of Azure-to-Logstash configuration. +A Blob Storage account is a central location that enables multiple instances of +Logstash to work together to process events. It records the +offset (location) of processed events. On restart, Logstash resumes processing +exactly where it left off. + +Configuration notes: + +* A Blob Storage account is highly recommended for use with this plugin, and is +likely required for production servers. +* The `storage_connection` option passes the blob storage connection string. +* Configure all Logstash instances to use the same `storage_connection` to get the +benefits of shared processing. + +Sample Blob Storage connection string: + +[source,text] +---- +DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net +---- + +Find the connection string to Blob Storage here: +https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. + +[id="{version}-plugins-{type}s-{plugin}-best-practices"] +===== Best practices + +Here are some guidelines to help you avoid data conflicts that can cause lost +events. + +* <<{version}-plugins-{type}s-{plugin}-bp-group>> +* <<{version}-plugins-{type}s-{plugin}-bp-multihub>> +* <<{version}-plugins-{type}s-{plugin}-bp-threads>> + +[id="{version}-plugins-{type}s-{plugin}-bp-group"] +===== Create a Logstash consumer group +Create a new consumer group specifically for Logstash. Do not use the $default or +any other consumer group that might already be in use. Reusing consumer groups +among non-related consumers can cause unexpected behavior and possibly lost +events. All Logstash instances should use the same consumer group so that they can +work together for processing events. + +[id="{version}-plugins-{type}s-{plugin}-bp-multihub"] +===== Avoid overwriting offset with multiple Event Hubs +The offsets (position) of the Event Hubs are stored in the configured Azure Blob +store. The Azure Blob store uses paths like a file system to store the offsets. +If the paths between multiple Event Hubs overlap, then the offsets may be stored +incorrectly. + +To avoid duplicate file paths, use the advanced configuration model and make +sure that at least one of these options is different per Event Hub: + +** storage_connection +** storage_container (defaults to Event Hub name if not defined) +** consumer_group + + +[id="{version}-plugins-{type}s-{plugin}-bp-threads"] +===== Set number of threads correctly + +By default, the number of threads used to service all event hubs is `16`. And while this +may be sufficient for most use cases, throughput may be improved by refining this number. +When servicing a large number of partitions across one or more event hubs, setting a higher +value may result in improved performance. The maximum number of threads is not strictly bound +by the total number of partitions being serviced, but setting the value much higher than +that may mean that some threads are idle. + +NOTE: The number of threads *must* be greater than or equal to the number of Event hubs plus one. + +NOTE: Threads are currently available only as a global setting across all event hubs in a single `azure_event_hubs` +input definition. However if your configuration includes multiple `azure_event_hubs` inputs, the threads setting applies +independently to each. + +**Example: Single event hub** + +If you’re collecting activity logs from one event hub instance, +then only 2 threads are required. + +* Event hubs = 1 +* Minimum threads = 2 (1 Event Hub + 1) + +**Example: Multiple event hubs** + +If you are collecting activity logs from more than event hub instance, then at least 1 thread per event hub is required. + +* Event hubs = 4 +* Minimum threads = 5 (4 Event Hubs + 1) + +When you are using multiple partitions per event hub, you may want to assign more threads. +A good base level is (1 + `number of event hubs * number of partitions`). +That is, one thread for each partition across all event hubs. + +[id="{version}-plugins-{type}s-{plugin}-eh_config_models"] +==== Configuration models + +This plugin supports two configuration models: basic and advanced. Basic +configuration is recommended for most use cases, and is illustrated in the +examples throughout this topic. + +[id="{version}-plugins-{type}s-{plugin}-eh_basic_config"] +===== Basic configuration (default) + +Basic configuration is the default and supports consuming from multiple Event +Hubs. All Events Hubs, except for the connection string, share the same +configuration. + +You supply a list of Event Hub connection strings, complete with the Event Hub +EntityPath that defines the Event Hub name. All other configuration settings are +shared. + +[source,ruby] +---- +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...EntityPath=insights-logs-errors", "Endpoint=sb://example2...EntityPath=insights-metrics-pt1m"] + threads => 8 + decorate_events => true + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-eh_advanced_config"] +===== Advanced configuration +The advanced configuration model accommodates deployments where different Event +Hubs require different configurations. Options can be configured per Event Hub. +You provide a list of Event Hub names through the `event_hubs` option. Under +each name, specify the configuration for that Event Hub. Options can be defined +globally or expressed per Event Hub. + +If the same configuration option appears in both the global and `event_hub` +sections, the more specific (event_hub) setting takes precedence. + +NOTE: Advanced configuration is not necessary or recommended for most use cases. + +[source,ruby] +---- +input { + azure_event_hubs { + config_mode => "advanced" + threads => 8 + decorate_events => true + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + initial_position => "beginning" + consumer_group => "iam_team" + }}, + {"insights-metrics-pt1m" => { + event_hub_connection => "Endpoint=sb://example2..." + initial_position => "end" + consumer_group => "db_team" + }} + ] + } +} +---- + +In this example, `storage_connection` and `decorate_events` are applied globally. +The two Event Hubs each have their own settings for `consumer_groups` +and `initial_position`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Azure Event Hubs Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-config_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], (`basic` or `advanced`)|No +| <<{version}-plugins-{type}s-{plugin}-event_hubs>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connections>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes, when `config_mode => basic` +| <<{version}-plugins-{type}s-{plugin}-event_hub_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes, when `config_mode => advanced` +| <<{version}-plugins-{type}s-{plugin}-checkpoint_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_group>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-initial_position>> |{logstash-ref}/configuration-file-structure.html#string[string], (`beginning`, `end`, or `look_back`)|No +| <<{version}-plugins-{type}s-{plugin}-initial_position_look_back>> |{logstash-ref}/configuration-file-structure.html#number[number]|No, unless `initial_position => look_back` +| <<{version}-plugins-{type}s-{plugin}-max_batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-storage_connection>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_container>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +NOTE: All Event Hubs options are common to both basic and advanced +configurations, with the following exceptions. The basic configuration uses +`event_hub_connections` to support multiple connections. The advanced +configuration uses `event_hubs` and `event_hub_connection` (singular). + +[id="{version}-plugins-{type}s-{plugin}-config_mode"] +===== `config_mode` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid entries are `basic` or `advanced` +* Default value is `basic` + +Sets configuration to either <<{version}-plugins-{type}s-{plugin}-eh_basic_config>> or <<{version}-plugins-{type}s-{plugin}-eh_advanced_config>>. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hubs"] +===== `event_hubs` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Ignored for basic configuration +* Required for advanced configuration + +Defines the Event Hubs to be read. An array of hashes where each entry is a +hash of the Event Hub name and its configuration options. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1..." + }}, + { "event_hub_name2" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" + }} + ] + consumer_group => "logstash" # shared across all Event Hubs +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connections"] +===== `event_hub_connections` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* No default value +* Required for basic configuration + +List of connection strings that identifies the Event Hubs to be read. Connection +strings include the EntityPath for the Event Hub. + +The `event_hub_connections` option is defined +per Event Hub. All other configuration options are shared among Event Hubs. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1" , "Endpoint=sb://example2...;EntityPath=event_hub_name2" ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-event_hub_connection"] +===== `event_hub_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value +* Valid only for advanced configuration + +Connection string that identifies the Event Hub to be read. Advanced +configuration options can be set per Event Hub. This option modifies +`event_hub_name`, and should be nested under it. (See sample.) This option +accepts only one connection string. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + event_hubs => [ + { "event_hub_name1" => { + event_hub_connection => "Endpoint=sb://example1...;EntityPath=event_hub_name1" + }} + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-checkpoint_interval"] +===== `checkpoint_interval` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `5` seconds +* Set to `0` to disable. + +Interval in seconds to write checkpoints during batch processing. Checkpoints +tell Logstash where to resume processing after a restart. Checkpoints are +automatically written at the end of each batch, regardless of this setting. + +Writing checkpoints too frequently can slow down processing unnecessarily. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + checkpoint_interval => 5 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-consumer_group"] +===== `consumer_group` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `$Default` + +Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that +consumer group so that they can work together properly. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + consumer_group => "logstash" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Adds metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + decorate_events => true +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position"] +===== `initial_position` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Valid arguments are `beginning`, `end`, `look_back` +* Default value is `beginning` + +When first reading from an Event Hub, start from this position: + +* `beginning` reads all pre-existing events in the Event Hub +* `end` does not read any pre-existing events in the Event Hub +* `look_back` reads `end` minus a number of seconds worth of pre-existing events. +You control the number of seconds using the `initial_position_look_back` option. + +Note: If `storage_connection` is set, the `initial_position` value is used only +the first time Logstash reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "beginning" +} +---- + +[id="{version}-plugins-{type}s-{plugin}-initial_position_look_back"] +===== `initial_position_look_back` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `86400` +* Used only if `initial_position` is set to `look-back` + +Number of seconds to look back to find the initial position for pre-existing +events. This option is used only if `initial_position` is set to `look_back`. If +`storage_connection` is set, this configuration applies only the first time Logstash +reads from the Event Hub. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + initial_position => "look_back" + initial_position_look_back => 86400 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-max_batch_size"] +===== `max_batch_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `125` + +Maximum number of events retrieved and processed together. A checkpoint is +created after each batch. Increasing this value may help with performance, but +requires more memory. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + max_batch_size => 125 +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_connection"] +===== `storage_connection` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* No default value + +Connection string for blob account storage. Blob account storage persists the +offsets between restarts, and ensures that multiple instances of Logstash +process different partitions. +When this value is set, restarts resume where processing left off. +When this value is not set, the `initial_position` value is used on every restart. + +We strongly recommend that you define this value for production environments. + + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." +} +---- + +[id="{version}-plugins-{type}s-{plugin}-storage_container"] +===== `storage_container` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Defaults to the Event Hub name if not defined + +Name of the storage container used to persist offsets and allow multiple instances of Logstash +to work together. + +[source,ruby] +---- +azure_event_hubs { + event_hub_connections => ["Endpoint=sb://example1...;EntityPath=event_hub_name1"] + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + storage_container => "my_container" +} +---- + +To avoid overwriting offsets, you can use different storage containers. This is +particularly important if you are monitoring two Event Hubs with the same name. +You can use the advanced configuration model to configure different storage +containers. + +[source,ruby] +---- +azure_event_hubs { + config_mode => "advanced" + consumer_group => "logstash" + storage_connection => "DefaultEndpointsProtocol=https;AccountName=example...." + event_hubs => [ + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example1..." + storage_container => "insights-operational-logs-1" + }}, + {"insights-operational-logs" => { + event_hub_connection => "Endpoint=sb://example2..." + storage_container => "insights-operational-logs-2" + }} + ] + } +---- + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Minimum value is `2` +* Default value is `16` + +Total number of threads used to process events. The value you set here applies +to all Event Hubs. Even with advanced configuration, this value is a global +setting, and can't be set per event hub. + +[source,ruby] +---- +azure_event_hubs { + threads => 16 +} +---- + +The number of threads should be the number of Event Hubs plus one or more. +See <<{version}-plugins-{type}s-{plugin}-best-practices>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-index.asciidoc b/docs/versioned-plugins/inputs/beats-index.asciidoc index a319e0cdd..34d29ad7e 100644 --- a/docs/versioned-plugins/inputs/beats-index.asciidoc +++ b/docs/versioned-plugins/inputs/beats-index.asciidoc @@ -5,6 +5,20 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-09-04 +| <> | 2025-02-12 +| <> | 2025-01-30 +| <> | 2024-12-02 +| <> | 2025-09-04 +| <> | 2025-02-12 +| <> | 2025-02-04 +| <> | 2024-11-18 +| <> | 2024-08-28 +| <> | 2024-08-28 +| <> | 2024-04-23 +| <> | 2024-03-13 +| <> | 2024-02-22 +| <> | 2024-01-23 | <> | 2023-11-20 | <> | 2023-10-18 | <> | 2023-09-07 @@ -97,6 +111,20 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-05 |======================================================================= +include::beats-v7.0.3.asciidoc[] +include::beats-v7.0.2.asciidoc[] +include::beats-v7.0.1.asciidoc[] +include::beats-v7.0.0.asciidoc[] +include::beats-v6.9.4.asciidoc[] +include::beats-v6.9.3.asciidoc[] +include::beats-v6.9.2.asciidoc[] +include::beats-v6.9.1.asciidoc[] +include::beats-v6.9.0.asciidoc[] +include::beats-v6.8.4.asciidoc[] +include::beats-v6.8.3.asciidoc[] +include::beats-v6.8.2.asciidoc[] +include::beats-v6.8.1.asciidoc[] +include::beats-v6.8.0.asciidoc[] include::beats-v6.7.2.asciidoc[] include::beats-v6.7.1.asciidoc[] include::beats-v6.7.0.asciidoc[] diff --git a/docs/versioned-plugins/inputs/beats-v6.8.0.asciidoc b/docs/versioned-plugins/inputs/beats-v6.8.0.asciidoc new file mode 100644 index 000000000..449707426 --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.8.0.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.8.0 +:release_date: 2024-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.8.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.8.1.asciidoc b/docs/versioned-plugins/inputs/beats-v6.8.1.asciidoc new file mode 100644 index 000000000..cc0625e3a --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.8.1.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.8.1 +:release_date: 2024-02-22 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.8.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.8.2.asciidoc b/docs/versioned-plugins/inputs/beats-v6.8.2.asciidoc new file mode 100644 index 000000000..656d5511a --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.8.2.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.8.2 +:release_date: 2024-03-13 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.8.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.8.3.asciidoc b/docs/versioned-plugins/inputs/beats-v6.8.3.asciidoc new file mode 100644 index 000000000..6e024b5eb --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.8.3.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.8.3 +:release_date: 2024-04-23 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.8.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.8.4.asciidoc b/docs/versioned-plugins/inputs/beats-v6.8.4.asciidoc new file mode 100644 index 000000000..8e1071c3e --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.8.4.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.8.4 +:release_date: 2024-08-28 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.8.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.9.0.asciidoc b/docs/versioned-plugins/inputs/beats-v6.9.0.asciidoc new file mode 100644 index 000000000..028db451f --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.9.0.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.9.0 +:release_date: 2024-08-28 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.9.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.9.1.asciidoc b/docs/versioned-plugins/inputs/beats-v6.9.1.asciidoc new file mode 100644 index 000000000..85061c677 --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.9.1.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.9.1 +:release_date: 2024-11-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.9.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.9.2.asciidoc b/docs/versioned-plugins/inputs/beats-v6.9.2.asciidoc new file mode 100644 index 000000000..75330669c --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.9.2.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.9.2 +:release_date: 2025-02-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.9.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.9.3.asciidoc b/docs/versioned-plugins/inputs/beats-v6.9.3.asciidoc new file mode 100644 index 000000000..85d138fe0 --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.9.3.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.9.3 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.9.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v6.9.4.asciidoc b/docs/versioned-plugins/inputs/beats-v6.9.4.asciidoc new file mode 100644 index 000000000..30b39ea68 --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v6.9.4.asciidoc @@ -0,0 +1,573 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.9.4 +:release_date: 2025-09-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v6.9.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_peer_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_peer_metadata"] +===== `ssl_peer_metadata` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enables storing client certificate information in event's metadata. + +This option is only valid when <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[6.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default, the server doesn't do any client verification. If the <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +When mutual TLS is enabled (`peer` or `force_peer`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3 + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v7.0.0.asciidoc b/docs/versioned-plugins/inputs/beats-v7.0.0.asciidoc new file mode 100644 index 000000000..0823e9f02 --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v7.0.0.asciidoc @@ -0,0 +1,517 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.0 +:release_date: 2024-12-02 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v7.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Beats Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_peer_metadata |<<{version}-plugins-{type}s-{plugin}-enrich>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v7.0.1.asciidoc b/docs/versioned-plugins/inputs/beats-v7.0.1.asciidoc new file mode 100644 index 000000000..5a4e056bf --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v7.0.1.asciidoc @@ -0,0 +1,517 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.1 +:release_date: 2025-01-30 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v7.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Beats Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_peer_metadata |<<{version}-plugins-{type}s-{plugin}-enrich>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v7.0.2.asciidoc b/docs/versioned-plugins/inputs/beats-v7.0.2.asciidoc new file mode 100644 index 000000000..248516109 --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v7.0.2.asciidoc @@ -0,0 +1,517 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.2 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v7.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Beats Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_peer_metadata |<<{version}-plugins-{type}s-{plugin}-enrich>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/beats-v7.0.3.asciidoc b/docs/versioned-plugins/inputs/beats-v7.0.3.asciidoc new file mode 100644 index 000000000..ecda80a13 --- /dev/null +++ b/docs/versioned-plugins/inputs/beats-v7.0.3.asciidoc @@ -0,0 +1,517 @@ +:plugin: beats +:type: input +:default_codec: plain +:plugin-uc: Beats +:plugin-singular: Beat + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.3 +:release_date: 2025-09-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-beats/blob/v7.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {plugin-uc} input plugin {version} + +NOTE: The `input-elastic_agent` plugin is the next generation of the +`input-beats` plugin. +They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase]. + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input plugin enables Logstash to receive events from the +{plugin-uc} framework. + +The following example shows how to configure Logstash to listen on port +5044 for incoming {plugin-uc} connections and to index into Elasticsearch. + +//Example for Beats +ifeval::["{plugin}"=="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1> + } +} +----- +<1> `%{[@metadata][beat]}` sets the first part of the index name to the value +of the metadata field and `%{[@metadata][version]}` sets the second part to +the {plugin-singular} version. For example: metricbeat-6.1.6. + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. + +NOTE: If ILM is not being used, set `index` to +`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so +Logstash creates an index per day, based on the `@timestamp` value of the events +coming from {plugin-uc}. +endif::[] + +//Example for Elastic Agent +ifeval::["{plugin}"!="beats"] +["source","sh",subs="attributes"] +----- + +input { + {plugin} { + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + data_stream => "true" + } +} +----- + +Events indexed into Elasticsearch with the Logstash configuration shown here +will be similar to events directly indexed by {plugin-uc} into Elasticsearch. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-memory"] +===== Memory usage + +This plugin uses "off-heap" direct memory in addition to heap memory. +By default, a JVM's off-heap direct memory limit is the same as the heap size. +For example, setting `-Xmx10G` without setting the direct memory limit will allocate `10GB` for heap and an additional `10GB` for direct memory, for a total of `20GB` allocated. +You can set the amount of direct memory with `-XX:MaxDirectMemorySize` in {logstash-ref}/jvm-settings.html[Logstash JVM Settings]. +Consider setting direct memory to half of the heap size. +Setting direct memory too low decreases the performance of ingestion. + +NOTE: Be sure that heap and direct memory combined does not exceed the total memory available on the server to avoid an OutOfDirectMemoryError + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-multiline"] +===== Multi-line events + +If you are shipping events that span multiple lines, you need to use +the {filebeat-ref}/multiline-examples.html[configuration options available in +Filebeat] to handle multiline events before sending the event data to Logstash. +You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec +plugin] to handle multiline events. Doing so will result in the failure to start +Logstash. +endif::[] + + +//Content for Beats +ifeval::["{plugin}"=="beats"] +[id="{version}-plugins-{type}s-{plugin}-versioned-indexes"] +==== Versioned indices + +To minimize the impact of future schema changes on your existing indices and +mappings in Elasticsearch, configure the Elasticsearch output to write to +versioned indices. The pattern that you specify for the `index` setting +controls the index name: + +[source,yaml] +---- +index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" +---- + +`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of +the `beat` metadata field, for example, `filebeat`. +`%{[@metadata][version]}`:: Sets the second part of the name to the Beat +version, for example, +{logstash_version}+. +`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the +Logstash `@timestamp` field. + +This configuration results in daily index names like ++filebeat-{logstash_version}-{localdate}+. +endif::[] + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event enrichment and the Elastic Common Schema (ECS) + +When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. +You can use the <<{version}-plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories. + +The location of these enrichment fields depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled: + +- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure. +- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema. + + +.`source_metadata` +[cols="> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-add_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_inactivity_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enrich>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-event_loop_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-executor_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_codec_tag>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-add_hostname"] +===== `add_hostname` + + deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field. + +[id="{version}-plugins-{type}s-{plugin}-client_inactivity_timeout"] +===== `client_inactivity_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Close Idle clients after X seconds of inactivity. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`: structured connection metadata added under ECS v1 compliant namespaces + ** `v8`: structured connection metadata added under ECS v8 compliant namespaces + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Refer to <<{version}-plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-enrich"] +===== `enrich` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +** An {logstash-ref}/configuration-file-structure.html#array[array] can also be provided +** Configures which enrichments are applied to each event +** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments. +** Supported values are: ++ +[cols="2l,5"] +|======================================================================= +|Enrichment | Description + +| codec_metadata | Information about how the codec transformed a sequence of bytes into + this Event, such as _which_ codec was used. Also, if no codec is + explicitly specified, _excluding_ `codec_metadata` from `enrich` will + disable `ecs_compatibility` for this plugin. +| source_metadata | Information about the _source_ of the event, such as the IP address + of the inbound connection this input received the event from +| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from, + such as identity information from the SSL client certificate that was + presented when establishing a connection to this input +| all | _alias_ to include _all_ available enrichments (including additional + enrichments introduced in future versions of this plugin) +| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_ + defining codec with this option will not disable the `ecs_compatibility`, + instead it relies on pipeline or codec `ecs_compatibility` configuration. +|======================================================================= + + +**Example:** + +This configuration disables _all_ enrichments: + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => none + } +} +-------------------------------------------------- + +Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others): + + +["source",subs="attributes"] +-------------------------------------------------- +input { + beats { + port => 5044 + enrich => [source_metadata, ssl_peer_metadata] + } +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-event_loop_threads"] +===== `event_loop_threads` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Defaults to 0. + +When setting `0`, the actual default is `available_processors * 2` + +This is an expert-level setting, and generally should not need to be set +{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads. +The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks. +This configuration intends to limit or increase the number of threads to be created for the event loop. +See <<{version}-plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count. + +[id="{version}-plugins-{type}s-{plugin}-executor_threads"] +===== `executor_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is equal to the number of CPU cores (1 executor thread per CPU core). + +The number of threads to be used to process incoming {plugin-uc} requests. +By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores. +These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. +Parsing the Lumberjack protocol is offloaded to a dedicated thread pool. + +Generally you don't need to touch this setting. +In case you are sending very large events and observing "OutOfDirectMemory" exceptions, +you may want to reduce this number to half or 1/4 of the CPU cores. +This change reduces the number of threads decompressing batches of data into direct memory. +However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment, +either by increasing number of Logstash nodes or increasing the JVM's Direct Memory. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The IP address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-include_codec_tag"] +===== `include_codec_tag` + +deprecated[6.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-enrich>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +When mutual TLS is enabled (`required` or `optional`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Beats Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_peer_metadata |<<{version}-plugins-{type}s-{plugin}-enrich>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/cloudwatch-index.asciidoc b/docs/versioned-plugins/inputs/cloudwatch-index.asciidoc index e4caf3648..975e43011 100644 --- a/docs/versioned-plugins/inputs/cloudwatch-index.asciidoc +++ b/docs/versioned-plugins/inputs/cloudwatch-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -24,6 +28,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::cloudwatch-v7.2.1.asciidoc[] +include::cloudwatch-v7.2.0.asciidoc[] +include::cloudwatch-v7.1.8.asciidoc[] +include::cloudwatch-v7.1.7.asciidoc[] include::cloudwatch-v7.1.6.asciidoc[] include::cloudwatch-v7.1.5.asciidoc[] include::cloudwatch-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/inputs/cloudwatch-v7.1.7.asciidoc b/docs/versioned-plugins/inputs/cloudwatch-v7.1.7.asciidoc new file mode 100644 index 000000000..99fbb3616 --- /dev/null +++ b/docs/versioned-plugins/inputs/cloudwatch-v7.1.7.asciidoc @@ -0,0 +1,330 @@ +:integration: aws +:plugin: cloudwatch +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Pull events from the Amazon Web Services CloudWatch API. + +To use this plugin, you *must* have an AWS account, and the following policy + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy for EC2 metrics is as follows: +[source,json] + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Stmt1444715676000", + "Effect": "Allow", + "Action": [ + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics" + ], + "Resource": "*" + }, + { + "Sid": "Stmt1444716576170", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances" + ], + "Resource": "*" + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + +===== Configuration examples +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EC2" + metrics => [ "CPUUtilization" ] + filters => { "tag:Group" => "API-Production" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EBS" + metrics => ["VolumeQueueLength"] + filters => { "tag:Monitoring" => "Yes" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/RDS" + metrics => ["CPUUtilization", "CPUCreditUsage"] + filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "sqlserver_test2" + metrics => [ "Memory Available Bytes"] + filters => { + InstanceId => "i-xxxxxxxxxxx" + objectname => "Memory" + } + combined => true + interval => 600 + period => 300 + } + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-combined>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-filters>> |{logstash-ref}/configuration-file-structure.html#array[array]|See <<{version}-plugins-{type}s-{plugin}-filters,note>> +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metrics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-period>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statistics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-combined"] +===== `combined` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Use this for namespaces that need to combine the dimensions like S3 and SNS. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-filters"] +===== `filters` + + * This setting can be required or optional. See note below. + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +NOTE: This setting is optional when the namespace is `AWS/EC2`. Otherwise this is a required field. + +Specify the filters to apply when fetching resources. Follow the AWS convention: + +* Instances: { 'instance-id' => 'i-12344321' } +* Tags: { "tag:Environment" => "Production" } +* Volumes: { 'attachment.status' => 'attached' } + +Each namespace uniquely support certain dimensions. Please consult the documentation +to ensure you're using valid filters. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `900` + +Set how frequently CloudWatch should be queried + +The default, `900`, means check every 15 minutes. Setting this value too low +(generally less than 300) results in no metrics being returned from CloudWatch. + +[id="{version}-plugins-{type}s-{plugin}-metrics"] +===== `metrics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"]` + +Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for the available metrics for other namespaces. + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"AWS/EC2"` + +If undefined, LogStash will complain, even if codec is unused. +The service namespace of the metrics to fetch. + +The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for valid values. + +[id="{version}-plugins-{type}s-{plugin}-period"] +===== `period` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300` + +Set the granularity of the returned datapoints. + +Must be at least 60 seconds and in multiples of 60. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-statistics"] +===== `statistics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["SampleCount", "Average", "Minimum", "Maximum", "Sum"]` + +Specify the statistics to fetch for each namespace + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-use_ssl"] +===== `use_ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Make sure we require the V1 classes when including this module. +require 'aws-sdk' will load v2 classes. +Should we require (true) or disable (false) using SSL for communicating with the AWS API +The AWS SDK for Ruby defaults to SSL so we preserve that + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/cloudwatch-v7.1.8.asciidoc b/docs/versioned-plugins/inputs/cloudwatch-v7.1.8.asciidoc new file mode 100644 index 000000000..438c930f4 --- /dev/null +++ b/docs/versioned-plugins/inputs/cloudwatch-v7.1.8.asciidoc @@ -0,0 +1,330 @@ +:integration: aws +:plugin: cloudwatch +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Pull events from the Amazon Web Services CloudWatch API. + +To use this plugin, you *must* have an AWS account, and the following policy + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy for EC2 metrics is as follows: +[source,json] + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Stmt1444715676000", + "Effect": "Allow", + "Action": [ + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics" + ], + "Resource": "*" + }, + { + "Sid": "Stmt1444716576170", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances" + ], + "Resource": "*" + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + +===== Configuration examples +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EC2" + metrics => [ "CPUUtilization" ] + filters => { "tag:Group" => "API-Production" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EBS" + metrics => ["VolumeQueueLength"] + filters => { "tag:Monitoring" => "Yes" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/RDS" + metrics => ["CPUUtilization", "CPUCreditUsage"] + filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "sqlserver_test2" + metrics => [ "Memory Available Bytes"] + filters => { + InstanceId => "i-xxxxxxxxxxx" + objectname => "Memory" + } + combined => true + interval => 600 + period => 300 + } + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-combined>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-filters>> |{logstash-ref}/configuration-file-structure.html#array[array]|See <<{version}-plugins-{type}s-{plugin}-filters,note>> +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metrics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-period>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statistics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-combined"] +===== `combined` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Use this for namespaces that need to combine the dimensions like S3 and SNS. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-filters"] +===== `filters` + + * This setting can be required or optional. See note below. + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +NOTE: This setting is optional when the namespace is `AWS/EC2`. Otherwise this is a required field. + +Specify the filters to apply when fetching resources. Follow the AWS convention: + +* Instances: { 'instance-id' => 'i-12344321' } +* Tags: { "tag:Environment" => "Production" } +* Volumes: { 'attachment.status' => 'attached' } + +Each namespace uniquely support certain dimensions. Please consult the documentation +to ensure you're using valid filters. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `900` + +Set how frequently CloudWatch should be queried + +The default, `900`, means check every 15 minutes. Setting this value too low +(generally less than 300) results in no metrics being returned from CloudWatch. + +[id="{version}-plugins-{type}s-{plugin}-metrics"] +===== `metrics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"]` + +Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for the available metrics for other namespaces. + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"AWS/EC2"` + +If undefined, LogStash will complain, even if codec is unused. +The service namespace of the metrics to fetch. + +The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for valid values. + +[id="{version}-plugins-{type}s-{plugin}-period"] +===== `period` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300` + +Set the granularity of the returned datapoints. + +Must be at least 60 seconds and in multiples of 60. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-statistics"] +===== `statistics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["SampleCount", "Average", "Minimum", "Maximum", "Sum"]` + +Specify the statistics to fetch for each namespace + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-use_ssl"] +===== `use_ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Make sure we require the V1 classes when including this module. +require 'aws-sdk' will load v2 classes. +Should we require (true) or disable (false) using SSL for communicating with the AWS API +The AWS SDK for Ruby defaults to SSL so we preserve that + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/cloudwatch-v7.2.0.asciidoc b/docs/versioned-plugins/inputs/cloudwatch-v7.2.0.asciidoc new file mode 100644 index 000000000..2492bf587 --- /dev/null +++ b/docs/versioned-plugins/inputs/cloudwatch-v7.2.0.asciidoc @@ -0,0 +1,342 @@ +:integration: aws +:plugin: cloudwatch +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Pull events from the Amazon Web Services CloudWatch API. + +To use this plugin, you *must* have an AWS account, and the following policy + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy for EC2 metrics is as follows: +[source,json] + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Stmt1444715676000", + "Effect": "Allow", + "Action": [ + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics" + ], + "Resource": "*" + }, + { + "Sid": "Stmt1444716576170", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances" + ], + "Resource": "*" + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + +===== Configuration examples +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EC2" + metrics => [ "CPUUtilization" ] + filters => { "tag:Group" => "API-Production" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EBS" + metrics => ["VolumeQueueLength"] + filters => { "tag:Monitoring" => "Yes" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/RDS" + metrics => ["CPUUtilization", "CPUCreditUsage"] + filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "sqlserver_test2" + metrics => [ "Memory Available Bytes"] + filters => { + InstanceId => "i-xxxxxxxxxxx" + objectname => "Memory" + } + combined => true + interval => 600 + period => 300 + } + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-combined>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-filters>> |{logstash-ref}/configuration-file-structure.html#array[array]|See <<{version}-plugins-{type}s-{plugin}-filters,note>> +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metrics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-period>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statistics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-combined"] +===== `combined` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Use this for namespaces that need to combine the dimensions like S3 and SNS. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-filters"] +===== `filters` + + * This setting can be required or optional. See note below. + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +NOTE: This setting is optional when the namespace is `AWS/EC2`. Otherwise this is a required field. + +Specify the filters to apply when fetching resources. Follow the AWS convention: + +* Instances: { 'instance-id' => 'i-12344321' } +* Tags: { "tag:Environment" => "Production" } +* Volumes: { 'attachment.status' => 'attached' } + +Each namespace uniquely support certain dimensions. Please consult the documentation +to ensure you're using valid filters. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `900` + +Set how frequently CloudWatch should be queried + +The default, `900`, means check every 15 minutes. Setting this value too low +(generally less than 300) results in no metrics being returned from CloudWatch. + +[id="{version}-plugins-{type}s-{plugin}-metrics"] +===== `metrics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"]` + +Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for the available metrics for other namespaces. + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"AWS/EC2"` + +If undefined, LogStash will complain, even if codec is unused. +The service namespace of the metrics to fetch. + +The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for valid values. + +[id="{version}-plugins-{type}s-{plugin}-period"] +===== `period` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300` + +Set the granularity of the returned datapoints. + +Must be at least 60 seconds and in multiples of 60. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-statistics"] +===== `statistics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["SampleCount", "Average", "Minimum", "Maximum", "Sum"]` + +Specify the statistics to fetch for each namespace + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-use_ssl"] +===== `use_ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Make sure we require the V1 classes when including this module. +require 'aws-sdk' will load v2 classes. +Should we require (true) or disable (false) using SSL for communicating with the AWS API +The AWS SDK for Ruby defaults to SSL so we preserve that + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/cloudwatch-v7.2.1.asciidoc b/docs/versioned-plugins/inputs/cloudwatch-v7.2.1.asciidoc new file mode 100644 index 000000000..c03abdc36 --- /dev/null +++ b/docs/versioned-plugins/inputs/cloudwatch-v7.2.1.asciidoc @@ -0,0 +1,342 @@ +:integration: aws +:plugin: cloudwatch +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Pull events from the Amazon Web Services CloudWatch API. + +To use this plugin, you *must* have an AWS account, and the following policy + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy for EC2 metrics is as follows: +[source,json] + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Stmt1444715676000", + "Effect": "Allow", + "Action": [ + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics" + ], + "Resource": "*" + }, + { + "Sid": "Stmt1444716576170", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances" + ], + "Resource": "*" + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + +===== Configuration examples +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EC2" + metrics => [ "CPUUtilization" ] + filters => { "tag:Group" => "API-Production" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/EBS" + metrics => ["VolumeQueueLength"] + filters => { "tag:Monitoring" => "Yes" } + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "AWS/RDS" + metrics => ["CPUUtilization", "CPUCreditUsage"] + filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier + region => "us-east-1" + } + } + +[source, ruby] + input { + cloudwatch { + namespace => "sqlserver_test2" + metrics => [ "Memory Available Bytes"] + filters => { + InstanceId => "i-xxxxxxxxxxx" + objectname => "Memory" + } + combined => true + interval => 600 + period => 300 + } + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-combined>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-filters>> |{logstash-ref}/configuration-file-structure.html#array[array]|See <<{version}-plugins-{type}s-{plugin}-filters,note>> +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metrics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-period>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statistics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-combined"] +===== `combined` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Use this for namespaces that need to combine the dimensions like S3 and SNS. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-filters"] +===== `filters` + + * This setting can be required or optional. See note below. + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +NOTE: This setting is optional when the namespace is `AWS/EC2`. Otherwise this is a required field. + +Specify the filters to apply when fetching resources. Follow the AWS convention: + +* Instances: { 'instance-id' => 'i-12344321' } +* Tags: { "tag:Environment" => "Production" } +* Volumes: { 'attachment.status' => 'attached' } + +Each namespace uniquely support certain dimensions. Please consult the documentation +to ensure you're using valid filters. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `900` + +Set how frequently CloudWatch should be queried + +The default, `900`, means check every 15 minutes. Setting this value too low +(generally less than 300) results in no metrics being returned from CloudWatch. + +[id="{version}-plugins-{type}s-{plugin}-metrics"] +===== `metrics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"]` + +Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for the available metrics for other namespaces. + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"AWS/EC2"` + +If undefined, LogStash will complain, even if codec is unused. +The service namespace of the metrics to fetch. + +The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html +for valid values. + +[id="{version}-plugins-{type}s-{plugin}-period"] +===== `period` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300` + +Set the granularity of the returned datapoints. + +Must be at least 60 seconds and in multiples of 60. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-statistics"] +===== `statistics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["SampleCount", "Average", "Minimum", "Maximum", "Sum"]` + +Specify the statistics to fetch for each namespace + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-use_ssl"] +===== `use_ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Make sure we require the V1 classes when including this module. +require 'aws-sdk' will load v2 classes. +Should we require (true) or disable (false) using SSL for communicating with the AWS API +The AWS SDK for Ruby defaults to SSL so we preserve that + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/dead_letter_queue-index.asciidoc b/docs/versioned-plugins/inputs/dead_letter_queue-index.asciidoc index a3223eb4f..f4acbd581 100644 --- a/docs/versioned-plugins/inputs/dead_letter_queue-index.asciidoc +++ b/docs/versioned-plugins/inputs/dead_letter_queue-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-09-04 | <> | 2022-08-02 | <> | 2022-06-16 | <> | 2022-03-01 @@ -24,6 +25,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-06 |======================================================================= +include::dead_letter_queue-v2.0.1.asciidoc[] include::dead_letter_queue-v2.0.0.asciidoc[] include::dead_letter_queue-v1.1.12.asciidoc[] include::dead_letter_queue-v1.1.11.asciidoc[] diff --git a/docs/versioned-plugins/inputs/dead_letter_queue-v2.0.1.asciidoc b/docs/versioned-plugins/inputs/dead_letter_queue-v2.0.1.asciidoc new file mode 100644 index 000000000..06b239207 --- /dev/null +++ b/docs/versioned-plugins/inputs/dead_letter_queue-v2.0.1.asciidoc @@ -0,0 +1,126 @@ +:plugin: dead_letter_queue +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v2.0.1 +:release_date: 2024-09-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-dead_letter_queue/blob/v2.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Dead_letter_queue input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Logstash input to read events from Logstash's dead letter queue. + +[source, sh] +----------------------------------------- +input { + dead_letter_queue { + path => "/var/logstash/data/dead_letter_queue" + start_timestamp => "2017-04-04T23:40:37" + } +} +----------------------------------------- + + +For more information about processing events in the dead letter queue, see +{logstash-ref}/dead-letter-queues.html[Dead Letter Queues]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Dead_letter_queue Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_consumed>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-commit_offsets>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |a valid filesystem path|Yes +| <<{version}-plugins-{type}s-{plugin}-pipeline_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sincedb_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-start_timestamp>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_consumed"] +===== `clean_consumed` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, this option deletes the DLQ segments that have been read. +This feature requires that `commit_offsets` is set to `true`. If not, you'll get a configuration error. +This feature is available in Logstash 8.4.0 and later. If this setting is `true` and and you are using a Logstash version older than 8.4.0, then you'll get a configuration error. + +[id="{version}-plugins-{type}s-{plugin}-commit_offsets"] +===== `commit_offsets` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Specifies whether this input should commit offsets as it processes the events. +Typically you specify `false` when you want to iterate multiple times over the +events in the dead letter queue, but don't want to save state. This is when you +are exploring the events in the dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to the dead letter queue directory that was created by a Logstash instance. +This is the path from which "dead" events are read and is typically configured +in the original Logstash instance with the setting `path.dead_letter_queue`. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_id"] +===== `pipeline_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"main"` + +ID of the pipeline whose events you want to read from. + +[id="{version}-plugins-{type}s-{plugin}-sincedb_path"] +===== `sincedb_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path of the sincedb database file (keeps track of the current position of dead letter queue) that +will be written to disk. The default will write sincedb files to `/plugins/inputs/dead_letter_queue`. + +NOTE: This value must be a file path and not a directory path. + +[id="{version}-plugins-{type}s-{plugin}-start_timestamp"] +===== `start_timestamp` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Timestamp in ISO8601 format from when you want to start processing the events from. +For example, `2017-04-04T23:40:37`. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/inputs/elastic_serverless_forwarder-index.asciidoc b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-index.asciidoc index 112625aac..8ad94bdb2 100644 --- a/docs/versioned-plugins/inputs/elastic_serverless_forwarder-index.asciidoc +++ b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-12-23 +| <> | 2024-12-18 +| <> | 2024-09-12 | <> | 2023-11-13 | <> | 2023-09-14 | <> | 2023-02-10 @@ -12,6 +15,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2022-11-18 |======================================================================= +include::elastic_serverless_forwarder-v2.0.0.asciidoc[] +include::elastic_serverless_forwarder-v1.0.0.asciidoc[] +include::elastic_serverless_forwarder-v0.1.5.asciidoc[] include::elastic_serverless_forwarder-v0.1.4.asciidoc[] include::elastic_serverless_forwarder-v0.1.3.asciidoc[] include::elastic_serverless_forwarder-v0.1.2.asciidoc[] diff --git a/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v0.1.5.asciidoc b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v0.1.5.asciidoc new file mode 100644 index 000000000..96c1afb52 --- /dev/null +++ b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v0.1.5.asciidoc @@ -0,0 +1,369 @@ +:plugin: elastic_serverless_forwarder +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.5 +:release_date: 2024-09-12 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elastic_serverless_forwarder/blob/v0.1.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:esf-name: Elastic Serverless Forwarder + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {esf-name} input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive events from {esf-name} over http(s) connections to the configured <<{version}-plugins-{type}s-{plugin}-port>>. + +[id="{version}-plugins-{type}s-{plugin}-ext-field"] +===== Minimum Configuration +[cols="3a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +input { + elastic_serverless_forwarder { + port => 8080 + ssl_certificate => "/path/to/logstash.crt" + ssl_key => "/path/to/logstash.key" + } +} +---- + +| + +[source,shell] +---- +input { + elastic_serverless_forwarder { + port => 8080 + ssl_enabled => false + } +} +---- + +|======================================================================================================================= + +.Technical Preview +**** +This {esf-name} input plugin is part of a _Technical Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to this plugin's https://github.com/logstash-plugins/logstash-input-elastic_serverless_forwarder/blob/main/CHANGELOG.md[CHANGELOG.md] to avoid being caught by surprise. +**** + + +[id="{version}-plugins-{type}s-{plugin}-enrichment"] +==== Enrichment + +This input provides _minimal enrichment_ on events, and avoids including information about itself, the client from which it received the data, or about the original event as-decoded from the request. + +NOTE: Senders are advised to use care with respect to fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash]. + ESF sends the Logstash-required `@timestamp` field by default, but if this value is missing it will be populated with the current time. + + +//// +// BEGIN: Elastic-internal implementation details +// + +[id="{version}-plugins-{type}s-{plugin}-blocking"] +==== Blocking Behavior + +When the Logstash pipeline that is receiving events from this input has a full queue, it prevents the pipeline's inputs from putting additional events into the queue until the capacity to do so becomes available. +This may result in an HTTP request timing out with some or all of its events unprocessed. +A client that receives an HTTP request timeout is expected to retry the entire request, even though it may cause duplication of data. + +When this plugin is blocked, it will reject _new_ requests with HTTP `429 Too Many Requests`. +Clients that receive `429`-s are expected to wait a moment before retrying the request — exponential back-off is encouraged to ease flood recovery. + +// +// END: Elastic-internal implementation details +//// + +[id="{version}-plugins-{type}s-{plugin}-security"] +==== Security + +This plugin has SSL on-by-default. + +At a minimum, you will need to either configure the plugin to present its identity, or disable SSL. + +Additionally, you may wish to authenticate clients using SSL client authentication, and/or authenticate requests using HTTP Basic authentication as described below. + +===== SSL Identity + +In order to establish SSL with a client, this input plugin will need to present an SSL certificate that the client trusts, and have access to the associated key. +These are configurable with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>, <<{version}-plugins-{type}s-{plugin}-ssl_key>>, and optionally <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>>. + +===== SSL Client Authentication + +By default, this plugin does not request certificates from clients during SSL negotiation. + +It can be configured to either request or require client certificates using <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>, +which often also requires configuring it with a list of <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> to trust. +When validating a certificate that is presented, <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> controls how certificates are verified. + +NOTE: ESF does not currently support _presenting_ client certificates, so requesting or requiring clients to present identity is only useful when combined with an SSL-terminating proxy. + +===== SSL Advanced Configuration + +This plugin exposes several advanced SSL configurations: + +* <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +* <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +* <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> + +===== HTTP Basic Authentication + +You can configure this plugin to authenticate requests using HTTP Basic authentication by configuring <<{version}-plugins-{type}s-{plugin}-auth_basic_username>> and <<{version}-plugins-{type}s-{plugin}-auth_basic_password>>. + +NOTE: Basic Authentication is not a substitute for SSL, as it provides neither secrecy nor security on its own. + When used with SSL disabled, HTTP Basic credentials are transmitted in effectively clear-text and can be easily recovered by an adversary. + +[id="{version}-plugins-{type}s-{plugin}-es-output-notes"] +==== Using {esf-name} with the Elasticsearch output +Here are some tips for configuring the {esf} input to work with the elasticsearch output: + +* Set the `document_id` in the output configuration when you use the {esf} input with an {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch[Elasticsearch output plugin]. ++ +[source,ruby] +---- +output { + elasticsearch { + ... + document_id => "%{[@metadata][_id]}" + ... + } +} +---- +* Starting from version 1.10.0 of {esf-name}, configuring `document_id` as shown in the example above is sufficient (the `_id` field is no longer available, and instead, Logstash now receives the `@metadata._id` field). + +* For {esf-name} v1.9.0 and earlier, rename the field `_id` to `@metadata._id` with a filter: ++ +[source,ruby] +---- +filter { + # support ESF < 1.10 + if [_id] and ![@metadata][_id] { + mutate { rename => { "_id" => "[@metadata][_id]" } } + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {esf-name} Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_basic_username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-auth_basic_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["certificate"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auth_basic_password"] +===== `auth_basic_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Password for HTTP basic authorization. +Requires <<{version}-plugins-{type}s-{plugin}-auth_basic_username>>. + +[id="{version}-plugins-{type}s-{plugin}-auth_basic_username"] +===== `auth_basic_username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Username for basic authorization. +Requires <<{version}-plugins-{type}s-{plugin}-auth_basic_password>>. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `"0.0.0.0"` (all available interfaces) + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[0.1.3, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Events are by default sent over SSL, which requires configuring this plugin to present an identity certificate using <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and key using <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +You can disable SSL with `+ssl => false+`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL certificate to use. +This certificate _MUST_ be PEM-formatted, and _MAY_ contain a chain of certificates starting with the certificate that identifies itself, followed by zero or more ordered intermediates optionally ending with the root signing authority. +Providing a complete chain allows clients to trust our certificate if their configuration allows them to trust one of our intermediates. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. + +If you wish to perform client authentication, you need to set `ssl_client_authentication` to `optional` or `required`. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. + +This is an advanced SSL configuration. + +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: +** `none`: do not request client's certificate, or validate certificates that are presented +** `optional`: request client's certificate, and validate it against our trust authorities _if-and-only-if_ it is presented +** `required`: require a valid certificate from the client that is signed by a trusted certificate authority +* Default value is `"none"` + +By default the server doesn't do any client authentication. +This means that connections from clients are _private_ when SSL is enabled, but that this input will allow SSL connections from _any_ client. +If you wish to configure this plugin to reject connections from untrusted hosts, you will need to configure this plugin to authenticate clients, and may also need to configure it with a list of `ssl_certificate_authorities`. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Events are, by default, sent over SSL, which requires configuring this plugin to present an identity certificate using <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and key using <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +You can disable SSL with `+ssl_enabled => false+`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +This is an advanced SSL configuration. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL key to use. + +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` +* Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. +`'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +This is an advanced SSL configuration. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is only one currently-supported mode: +** `certificate`: verifies that a certificate provided by the client is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key, but does _not_ perform hostname validation. +// NOTE: `ssl_verification_mode => full` is pending upstream support, but when it arrives it will look like: +//// +** `full`: fully validates the certificate as in `certificate` mode, and also validates that the client's outbound IP address is represented in the certificate's identity claims for `subject` and/or `subjectAltName`. +//// +* The default value is `certificate`. + +When <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> causes a client to present a certificate, this setting controls how that certificate is verified. + +NOTE: Client identity is not typically validated using SSL because the receiving server only has access to the client's outbound-ip, which is not always constant and is frequently not represented in the certificate's subject or subjectAltNames extensions. + For more information, see https://www.rfc-editor.org/rfc/rfc2818#section-3.1[RFC2818 § 3.2 (HTTP over TLS -- Client Identity)] + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v1.0.0.asciidoc b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v1.0.0.asciidoc new file mode 100644 index 000000000..c14425d0b --- /dev/null +++ b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v1.0.0.asciidoc @@ -0,0 +1,361 @@ +:plugin: elastic_serverless_forwarder +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.0 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elastic_serverless_forwarder/blob/v1.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:esf-name: Elastic Serverless Forwarder + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {esf-name} input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive events from {esf-name} over http(s) connections to the configured <<{version}-plugins-{type}s-{plugin}-port>>. + +[id="{version}-plugins-{type}s-{plugin}-ext-field"] +===== Minimum Configuration +[cols="3a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +input { + elastic_serverless_forwarder { + port => 8080 + ssl_certificate => "/path/to/logstash.crt" + ssl_key => "/path/to/logstash.key" + } +} +---- + +| + +[source,shell] +---- +input { + elastic_serverless_forwarder { + port => 8080 + ssl_enabled => false + } +} +---- + +|======================================================================================================================= + +[id="{version}-plugins-{type}s-{plugin}-enrichment"] +==== Enrichment + +This input provides _minimal enrichment_ on events, and avoids including information about itself, the client from which it received the data, or about the original event as-decoded from the request. + +NOTE: Senders are advised to use care with respect to fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash]. + ESF sends the Logstash-required `@timestamp` field by default, but if this value is missing it will be populated with the current time. + + +//// +// BEGIN: Elastic-internal implementation details +// + +[id="{version}-plugins-{type}s-{plugin}-blocking"] +==== Blocking Behavior + +When the Logstash pipeline that is receiving events from this input has a full queue, it prevents the pipeline's inputs from putting additional events into the queue until the capacity to do so becomes available. +This may result in an HTTP request timing out with some or all of its events unprocessed. +A client that receives an HTTP request timeout is expected to retry the entire request, even though it may cause duplication of data. + +When this plugin is blocked, it will reject _new_ requests with HTTP `429 Too Many Requests`. +Clients that receive `429`-s are expected to wait a moment before retrying the request — exponential back-off is encouraged to ease flood recovery. + +// +// END: Elastic-internal implementation details +//// + +[id="{version}-plugins-{type}s-{plugin}-security"] +==== Security + +This plugin has SSL on-by-default. + +At a minimum, you will need to either configure the plugin to present its identity, or disable SSL. + +Additionally, you may wish to authenticate clients using SSL client authentication, and/or authenticate requests using HTTP Basic authentication as described below. + +===== SSL Identity + +In order to establish SSL with a client, this input plugin will need to present an SSL certificate that the client trusts, and have access to the associated key. +These are configurable with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>, <<{version}-plugins-{type}s-{plugin}-ssl_key>>, and optionally <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>>. + +===== SSL Client Authentication + +By default, this plugin does not request certificates from clients during SSL negotiation. + +It can be configured to either request or require client certificates using <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>, +which often also requires configuring it with a list of <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> to trust. +When validating a certificate that is presented, <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> controls how certificates are verified. + +NOTE: ESF does not currently support _presenting_ client certificates, so requesting or requiring clients to present identity is only useful when combined with an SSL-terminating proxy. + +===== SSL Advanced Configuration + +This plugin exposes several advanced SSL configurations: + +* <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +* <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +* <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> + +===== HTTP Basic Authentication + +You can configure this plugin to authenticate requests using HTTP Basic authentication by configuring <<{version}-plugins-{type}s-{plugin}-auth_basic_username>> and <<{version}-plugins-{type}s-{plugin}-auth_basic_password>>. + +NOTE: Basic Authentication is not a substitute for SSL, as it provides neither secrecy nor security on its own. + When used with SSL disabled, HTTP Basic credentials are transmitted in effectively clear-text and can be easily recovered by an adversary. + +[id="{version}-plugins-{type}s-{plugin}-es-output-notes"] +==== Using {esf-name} with the Elasticsearch output +Here are some tips for configuring the {esf} input to work with the elasticsearch output: + +* Set the `document_id` in the output configuration when you use the {esf} input with an {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch[Elasticsearch output plugin]. ++ +[source,ruby] +---- +output { + elasticsearch { + ... + document_id => "%{[@metadata][_id]}" + ... + } +} +---- +* Starting from version 1.10.0 of {esf-name}, configuring `document_id` as shown in the example above is sufficient (the `_id` field is no longer available, and instead, Logstash now receives the `@metadata._id` field). + +* For {esf-name} v1.9.0 and earlier, rename the field `_id` to `@metadata._id` with a filter: ++ +[source,ruby] +---- +filter { + # support ESF < 1.10 + if [_id] and ![@metadata][_id] { + mutate { rename => { "_id" => "[@metadata][_id]" } } + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {esf-name} Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_basic_username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-auth_basic_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["certificate"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auth_basic_password"] +===== `auth_basic_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Password for HTTP basic authorization. +Requires <<{version}-plugins-{type}s-{plugin}-auth_basic_username>>. + +[id="{version}-plugins-{type}s-{plugin}-auth_basic_username"] +===== `auth_basic_username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Username for basic authorization. +Requires <<{version}-plugins-{type}s-{plugin}-auth_basic_password>>. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `"0.0.0.0"` (all available interfaces) + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[0.1.3, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Events are by default sent over SSL, which requires configuring this plugin to present an identity certificate using <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and key using <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +You can disable SSL with `+ssl => false+`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL certificate to use. +This certificate _MUST_ be PEM-formatted, and _MAY_ contain a chain of certificates starting with the certificate that identifies itself, followed by zero or more ordered intermediates optionally ending with the root signing authority. +Providing a complete chain allows clients to trust our certificate if their configuration allows them to trust one of our intermediates. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. + +If you wish to perform client authentication, you need to set `ssl_client_authentication` to `optional` or `required`. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. + +This is an advanced SSL configuration. + +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: +** `none`: do not request client's certificate, or validate certificates that are presented +** `optional`: request client's certificate, and validate it against our trust authorities _if-and-only-if_ it is presented +** `required`: require a valid certificate from the client that is signed by a trusted certificate authority +* Default value is `"none"` + +By default the server doesn't do any client authentication. +This means that connections from clients are _private_ when SSL is enabled, but that this input will allow SSL connections from _any_ client. +If you wish to configure this plugin to reject connections from untrusted hosts, you will need to configure this plugin to authenticate clients, and may also need to configure it with a list of `ssl_certificate_authorities`. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Events are, by default, sent over SSL, which requires configuring this plugin to present an identity certificate using <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and key using <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +You can disable SSL with `+ssl_enabled => false+`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +This is an advanced SSL configuration. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL key to use. + +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` +* Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. +`'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +This is an advanced SSL configuration. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is only one currently-supported mode: +** `certificate`: verifies that a certificate provided by the client is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key, but does _not_ perform hostname validation. +// NOTE: `ssl_verification_mode => full` is pending upstream support, but when it arrives it will look like: +//// +** `full`: fully validates the certificate as in `certificate` mode, and also validates that the client's outbound IP address is represented in the certificate's identity claims for `subject` and/or `subjectAltName`. +//// +* The default value is `certificate`. + +When <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> causes a client to present a certificate, this setting controls how that certificate is verified. + +NOTE: Client identity is not typically validated using SSL because the receiving server only has access to the client's outbound-ip, which is not always constant and is frequently not represented in the certificate's subject or subjectAltNames extensions. + For more information, see https://www.rfc-editor.org/rfc/rfc2818#section-3.1[RFC2818 § 3.2 (HTTP over TLS -- Client Identity)] + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v2.0.0.asciidoc b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v2.0.0.asciidoc new file mode 100644 index 000000000..6155fe5dd --- /dev/null +++ b/docs/versioned-plugins/inputs/elastic_serverless_forwarder-v2.0.0.asciidoc @@ -0,0 +1,364 @@ +:plugin: elastic_serverless_forwarder +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v2.0.0 +:release_date: 2024-12-23 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elastic_serverless_forwarder/blob/v2.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:esf-name: Elastic Serverless Forwarder + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {esf-name} input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive events from {esf-name} over http(s) connections to the configured <<{version}-plugins-{type}s-{plugin}-port>>. + +[id="{version}-plugins-{type}s-{plugin}-ext-field"] +===== Minimum Configuration +[cols="3a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +input { + elastic_serverless_forwarder { + port => 8080 + ssl_certificate => "/path/to/logstash.crt" + ssl_key => "/path/to/logstash.key" + } +} +---- + +| + +[source,shell] +---- +input { + elastic_serverless_forwarder { + port => 8080 + ssl_enabled => false + } +} +---- + +|======================================================================================================================= + +[id="{version}-plugins-{type}s-{plugin}-enrichment"] +==== Enrichment + +This input provides _minimal enrichment_ on events, and avoids including information about itself, the client from which it received the data, or about the original event as-decoded from the request. + +NOTE: Senders are advised to use care with respect to fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash]. + ESF sends the Logstash-required `@timestamp` field by default, but if this value is missing it will be populated with the current time. + + +//// +// BEGIN: Elastic-internal implementation details +// + +[id="{version}-plugins-{type}s-{plugin}-blocking"] +==== Blocking Behavior + +When the Logstash pipeline that is receiving events from this input has a full queue, it prevents the pipeline's inputs from putting additional events into the queue until the capacity to do so becomes available. +This may result in an HTTP request timing out with some or all of its events unprocessed. +A client that receives an HTTP request timeout is expected to retry the entire request, even though it may cause duplication of data. + +When this plugin is blocked, it will reject _new_ requests with HTTP `429 Too Many Requests`. +Clients that receive `429`-s are expected to wait a moment before retrying the request — exponential back-off is encouraged to ease flood recovery. + +// +// END: Elastic-internal implementation details +//// + +[id="{version}-plugins-{type}s-{plugin}-security"] +==== Security + +This plugin has SSL on-by-default. + +At a minimum, you will need to either configure the plugin to present its identity, or disable SSL. + +Additionally, you may wish to authenticate clients using SSL client authentication, and/or authenticate requests using HTTP Basic authentication as described below. + +===== SSL Identity + +In order to establish SSL with a client, this input plugin will need to present an SSL certificate that the client trusts, and have access to the associated key. +These are configurable with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>, <<{version}-plugins-{type}s-{plugin}-ssl_key>>, and optionally <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>>. + +===== SSL Client Authentication + +By default, this plugin does not request certificates from clients during SSL negotiation. + +It can be configured to either request or require client certificates using <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>, +which often also requires configuring it with a list of <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> to trust. +When validating a certificate that is presented, <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> controls how certificates are verified. + +NOTE: ESF does not currently support _presenting_ client certificates, so requesting or requiring clients to present identity is only useful when combined with an SSL-terminating proxy. + +===== SSL Advanced Configuration + +This plugin exposes several advanced SSL configurations: + +* <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +* <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +* <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> + +===== HTTP Basic Authentication + +You can configure this plugin to authenticate requests using HTTP Basic authentication by configuring <<{version}-plugins-{type}s-{plugin}-auth_basic_username>> and <<{version}-plugins-{type}s-{plugin}-auth_basic_password>>. + +NOTE: Basic Authentication is not a substitute for SSL, as it provides neither secrecy nor security on its own. + When used with SSL disabled, HTTP Basic credentials are transmitted in effectively clear-text and can be easily recovered by an adversary. + +[id="{version}-plugins-{type}s-{plugin}-es-output-notes"] +==== Using {esf-name} with the Elasticsearch output +Here are some tips for configuring the {esf} input to work with the elasticsearch output: + +* Set the `document_id` in the output configuration when you use the {esf} input with an {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch[Elasticsearch output plugin]. ++ +[source,ruby] +---- +output { + elasticsearch { + ... + document_id => "%{[@metadata][_id]}" + ... + } +} +---- +* Starting from version 1.10.0 of {esf-name}, configuring `document_id` as shown in the example above is sufficient (the `_id` field is no longer available, and instead, Logstash now receives the `@metadata._id` field). + +* For {esf-name} v1.9.0 and earlier, rename the field `_id` to `@metadata._id` with a filter: ++ +[source,ruby] +---- +filter { + # support ESF < 1.10 + if [_id] and ![@metadata][_id] { + mutate { rename => { "_id" => "[@metadata][_id]" } } + } +} +---- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {esf-name} Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `2.0.0` of this plugin, a previously deprecated SSL setting has been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_basic_username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-auth_basic_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["certificate"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auth_basic_password"] +===== `auth_basic_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Password for HTTP basic authorization. +Requires <<{version}-plugins-{type}s-{plugin}-auth_basic_username>>. + +[id="{version}-plugins-{type}s-{plugin}-auth_basic_username"] +===== `auth_basic_username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Username for basic authorization. +Requires <<{version}-plugins-{type}s-{plugin}-auth_basic_password>>. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `"0.0.0.0"` (all available interfaces) + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL certificate to use. +This certificate _MUST_ be PEM-formatted, and _MAY_ contain a chain of certificates starting with the certificate that identifies itself, followed by zero or more ordered intermediates optionally ending with the root signing authority. +Providing a complete chain allows clients to trust our certificate if their configuration allows them to trust one of our intermediates. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. + +If you wish to perform client authentication, you need to set `ssl_client_authentication` to `optional` or `required`. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. + +This is an advanced SSL configuration. + +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: +** `none`: do not request client's certificate, or validate certificates that are presented +** `optional`: request client's certificate, and validate it against our trust authorities _if-and-only-if_ it is presented +** `required`: require a valid certificate from the client that is signed by a trusted certificate authority +* Default value is `"none"` + +By default the server doesn't do any client authentication. +This means that connections from clients are _private_ when SSL is enabled, but that this input will allow SSL connections from _any_ client. +If you wish to configure this plugin to reject connections from untrusted hosts, you will need to configure this plugin to authenticate clients, and may also need to configure it with a list of `ssl_certificate_authorities`. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Events are, by default, sent over SSL, which requires configuring this plugin to present an identity certificate using <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and key using <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +You can disable SSL with `+ssl_enabled => false+`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +This is an advanced SSL configuration. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL key to use. + +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` +* Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. +`'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +This is an advanced SSL configuration. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is only one currently-supported mode: +** `certificate`: verifies that a certificate provided by the client is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key, but does _not_ perform hostname validation. +// NOTE: `ssl_verification_mode => full` is pending upstream support, but when it arrives it will look like: +//// +** `full`: fully validates the certificate as in `certificate` mode, and also validates that the client's outbound IP address is represented in the certificate's identity claims for `subject` and/or `subjectAltName`. +//// +* The default value is `certificate`. + +When <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> causes a client to present a certificate, this setting controls how that certificate is verified. + +NOTE: Client identity is not typically validated using SSL because the receiving server only has access to the client's outbound-ip, which is not always constant and is frequently not represented in the certificate's subject or subjectAltNames extensions. + For more information, see https://www.rfc-editor.org/rfc/rfc2818#section-3.1[RFC2818 § 3.2 (HTTP over TLS -- Client Identity)] + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `2.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-index.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-index.asciidoc index 18d55e5b3..1f86a8091 100644 --- a/docs/versioned-plugins/inputs/elasticsearch-index.asciidoc +++ b/docs/versioned-plugins/inputs/elasticsearch-index.asciidoc @@ -5,6 +5,22 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-06-06 +| <> | 2025-04-07 +| <> | 2025-03-17 +| <> | 2024-12-18 +| <> | 2025-06-06 +| <> | 2025-04-07 +| <> | 2025-03-17 +| <> | 2025-01-07 +| <> | 2024-12-18 +| <> | 2024-12-05 +| <> | 2024-08-23 +| <> | 2024-06-11 +| <> | 2024-03-05 +| <> | 2024-02-14 +| <> | 2024-01-24 +| <> | 2023-12-04 | <> | 2023-11-13 | <> | 2023-09-29 | <> | 2023-06-02 @@ -46,6 +62,22 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::elasticsearch-v5.2.0.asciidoc[] +include::elasticsearch-v5.1.0.asciidoc[] +include::elasticsearch-v5.0.2.asciidoc[] +include::elasticsearch-v5.0.0.asciidoc[] +include::elasticsearch-v4.23.0.asciidoc[] +include::elasticsearch-v4.22.0.asciidoc[] +include::elasticsearch-v4.21.2.asciidoc[] +include::elasticsearch-v4.21.1.asciidoc[] +include::elasticsearch-v4.21.0.asciidoc[] +include::elasticsearch-v4.20.5.asciidoc[] +include::elasticsearch-v4.20.4.asciidoc[] +include::elasticsearch-v4.20.3.asciidoc[] +include::elasticsearch-v4.20.2.asciidoc[] +include::elasticsearch-v4.20.1.asciidoc[] +include::elasticsearch-v4.20.0.asciidoc[] +include::elasticsearch-v4.19.1.asciidoc[] include::elasticsearch-v4.19.0.asciidoc[] include::elasticsearch-v4.18.0.asciidoc[] include::elasticsearch-v4.17.2.asciidoc[] diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.19.1.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.19.1.asciidoc new file mode 100644 index 000000000..e727c9f5f --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.19.1.asciidoc @@ -0,0 +1,649 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.19.1 +:release_date: 2023-12-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.19.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. See {ref}/multi-index.html[Multi Indices +documentation] in the Elasticsearch documentation for more information on how to +reference multiple indices. + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.20.0.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.20.0.asciidoc new file mode 100644 index 000000000..ca0291bbc --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.20.0.asciidoc @@ -0,0 +1,664 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.20.0 +:release_date: 2024-01-24 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.20.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. See {ref}/multi-index.html[Multi Indices +documentation] in the Elasticsearch documentation for more information on how to +reference multiple indices. + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.20.1.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.20.1.asciidoc new file mode 100644 index 000000000..c2a299c12 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.20.1.asciidoc @@ -0,0 +1,664 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.20.1 +:release_date: 2024-02-14 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.20.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. See {ref}/multi-index.html[Multi Indices +documentation] in the Elasticsearch documentation for more information on how to +reference multiple indices. + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.20.2.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.20.2.asciidoc new file mode 100644 index 000000000..9c2ed5b5c --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.20.2.asciidoc @@ -0,0 +1,664 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.20.2 +:release_date: 2024-03-05 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.20.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. See {ref}/multi-index.html[Multi Indices +documentation] in the Elasticsearch documentation for more information on how to +reference multiple indices. + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.20.3.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.20.3.asciidoc new file mode 100644 index 000000000..8ee6ede7f --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.20.3.asciidoc @@ -0,0 +1,664 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.20.3 +:release_date: 2024-06-11 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.20.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.20.4.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.20.4.asciidoc new file mode 100644 index 000000000..15dd730a0 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.20.4.asciidoc @@ -0,0 +1,664 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.20.4 +:release_date: 2024-08-23 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.20.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.20.5.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.20.5.asciidoc new file mode 100644 index 000000000..943e117c9 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.20.5.asciidoc @@ -0,0 +1,664 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.20.5 +:release_date: 2024-12-05 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.20.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.21.0.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.21.0.asciidoc new file mode 100644 index 000000000..20d312640 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.21.0.asciidoc @@ -0,0 +1,675 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.21.0 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.21.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.21.1.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.21.1.asciidoc new file mode 100644 index 000000000..0a0bb027d --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.21.1.asciidoc @@ -0,0 +1,685 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.21.1 +:release_date: 2025-01-07 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.21.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<{version}-plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace. + - When <<{version}-plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<{version}-plugins-{type}s-{plugin}-target>> and <<{version}-plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.21.2.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.21.2.asciidoc new file mode 100644 index 000000000..e3f6d690a --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.21.2.asciidoc @@ -0,0 +1,685 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.21.2 +:release_date: 2025-03-17 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.21.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<{version}-plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace. + - When <<{version}-plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<{version}-plugins-{type}s-{plugin}-target>> and <<{version}-plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.22.0.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.22.0.asciidoc new file mode 100644 index 000000000..0b48097a6 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.22.0.asciidoc @@ -0,0 +1,862 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.22.0 +:release_date: 2025-04-07 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.22.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + +[id="{version}-plugins-{type}s-{plugin}-scheduling"] +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<{version}-plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace. + - When <<{version}-plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<{version}-plugins-{type}s-{plugin}-target>> and <<{version}-plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict. + +[id="{version}-plugins-{type}s-{plugin}-cursor"] +==== Tracking a field's value across runs + +.Technical Preview: Tracking a field's value +**** +The feature that allows tracking a field's value across runs is in _Technical Preview_. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +Some uses cases require tracking the value of a particular field between two jobs. +Examples include: + +* avoiding the need to re-process the entire result set of a long query after an unplanned restart +* grabbing only new data from an index instead of processing the entire set on each job. + +The Elasticsearch input plugin provides the <<{version}-plugins-{type}s-{plugin}-tracking_field>> and <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> options. +When <<{version}-plugins-{type}s-{plugin}-tracking_field>> is set, the plugin records the value of that field for the last document retrieved in a run into +a file. +(The file location defaults to <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>>.) + +You can then inject this value in the query using the placeholder `:last_value`. +The value will be injected into the query before execution, and then updated after the query completes if new data was found. + +This feature works best when: + +* the query sorts by the tracking field, +* the timestamp field is added by {es}, and +* the field type has enough resolution so that two events are unlikely to have the same value. + +Consider using a tracking field whose type is https://www.elastic.co/guide/en/elasticsearch/reference/current/date_nanos.html[date nanoseconds]. +If the tracking field is of this data type, you can use an extra placeholder called `:present` to inject the nano-second based value of "now-30s". +This placeholder is useful as the right-hand side of a range filter, allowing the collection of +new data but leaving partially-searchable bulk request data to the next scheduled job. + +[id="{version}-plugins-{type}s-{plugin}-tracking-sample"] +===== Sample configuration: Track field value across runs + +This section contains a series of steps to help you set up the "tailing" of data being written to a set of indices, using a date nanosecond field added by an Elasticsearch ingest pipeline and the `tracking_field` capability of this plugin. + +. Create ingest pipeline that adds Elasticsearch's `_ingest.timestamp` field to the documents as `event.ingested`: ++ +[source, json] + PUT _ingest/pipeline/my-pipeline + { + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.putIfAbsent(\"event\", [:]); ctx.event.ingested = metadata().now.format(DateTimeFormatter.ISO_INSTANT);" + } + } + ] + } + +[start=2] +. Create an index mapping where the tracking field is of date nanosecond type and invokes the defined pipeline: ++ +[source, json] + PUT /_template/my_template + { + "index_patterns": ["test-*"], + "settings": { + "index.default_pipeline": "my-pipeline", + }, + "mappings": { + "properties": { + "event": { + "properties": { + "ingested": { + "type": "date_nanos", + "format": "strict_date_optional_time_nanos" + } + } + } + } + } + } + +[start=3] +. Define a query that looks at all data of the indices, sorted by the tracking field, and with a range filter since the last value seen until present: ++ +[source,json] +{ + "query": { + "range": { + "event.ingested": { + "gt": ":last_value", + "lt": ":present" + } + } + }, + "sort": [ + { + "event.ingested": { + "order": "asc", + "format": "strict_date_optional_time_nanos", + "numeric_type": "date_nanos" + } + } + ] +} + +[start=4] +. Configure the Elasticsearch input to query the indices with the query defined above, every minute, and track the `event.ingested` field: ++ +[source, ruby] + input { + elasticsearch { + id => tail_test_index + hosts => [ 'https://..'] + api_key => '....' + index => 'test-*' + query => '{ "query": { "range": { "event.ingested": { "gt": ":last_value", "lt": ":present"}}}, "sort": [ { "event.ingested": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" } } ] }' + tracking_field => "[event][ingested]" + slices => 5 # optional use of slices to speed data processing, should be equal to or less than number of primary shards + schedule => '* * * * *' # every minute + schedule_overlap => false # don't accumulate jobs if one takes longer than 1 minute + } + } + +With this sample setup, new documents are indexed into a `test-*` index. +The next scheduled run: + +* selects all new documents since the last observed value of the tracking field, +* uses {ref}/point-in-time-api.html#point-in-time-api[Point in time (PIT)] + {ref}/paginate-search-results.html#search-after[Search after] to paginate through all the data, and +* updates the value of the field at the end of the pagination. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schedule_overlap>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The path to store the last observed value of the tracking field, when used. +By default this file is stored as `/plugins/inputs/elasticsearch//last_run_value`. + +This setting should point to file, not a directory, and Logstash must have read+write access to this file. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-schedule_overlap"] +===== `schedule_overlap` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to allow queuing of a scheduled run if a run is occurring. +While this is ideal for ensuring a new run happens immediately after the previous on finishes if there +is a lot of work to do, but given the queue is unbounded it may lead to an out of memory over long periods of time +if the queue grows continuously. + +When in doubt, set `schedule_overlap` to false (it may become the default value in the future). + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + +[id="{version}-plugins-{type}s-{plugin}-tracking_field"] +===== `tracking_field` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Which field from the last event of a previous run will be used a cursor value for the following run. +The value of this field is injected into each query if the query uses the placeholder `:last_value`. +For the first query after a pipeline is started, the value used is either read from <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> file, +or taken from <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> setting. + +Note: The tracking value is updated after each page is read and at the end of each Point in Time. In case of a crash the last saved value will be used so some duplication of data can occur. For this reason the use of unique document IDs for each event is recommended in the downstream destination. + +[id="{version}-plugins-{type}s-{plugin}-tracking_field_seed"] +===== `tracking_field_seed` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `"1970-01-01T00:00:00.000000000Z"` + +The starting value for the <<{version}-plugins-{type}s-{plugin}-tracking_field>> if there is no <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> already. +This field defaults to the nanosecond precision ISO8601 representation of `epoch`, or "1970-01-01T00:00:00.000000000Z", given nano-second precision timestamps are the +most reliable data format to use for this feature. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v4.23.0.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v4.23.0.asciidoc new file mode 100644 index 000000000..7f00086e8 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v4.23.0.asciidoc @@ -0,0 +1,980 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.23.0 +:release_date: 2025-06-06 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v4.23.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + +[id="{version}-plugins-{type}s-{plugin}-scheduling"] +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<{version}-plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace. + - When <<{version}-plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<{version}-plugins-{type}s-{plugin}-target>> and <<{version}-plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict. + +[id="{version}-plugins-{type}s-{plugin}-cursor"] +==== Tracking a field's value across runs + +.Technical Preview: Tracking a field's value +**** +The feature that allows tracking a field's value across runs is in _Technical Preview_. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +Some uses cases require tracking the value of a particular field between two jobs. +Examples include: + +* avoiding the need to re-process the entire result set of a long query after an unplanned restart +* grabbing only new data from an index instead of processing the entire set on each job. + +The Elasticsearch input plugin provides the <<{version}-plugins-{type}s-{plugin}-tracking_field>> and <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> options. +When <<{version}-plugins-{type}s-{plugin}-tracking_field>> is set, the plugin records the value of that field for the last document retrieved in a run into +a file. +(The file location defaults to <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>>.) + +You can then inject this value in the query using the placeholder `:last_value`. +The value will be injected into the query before execution, and then updated after the query completes if new data was found. + +This feature works best when: + +* the query sorts by the tracking field, +* the timestamp field is added by {es}, and +* the field type has enough resolution so that two events are unlikely to have the same value. + +Consider using a tracking field whose type is https://www.elastic.co/guide/en/elasticsearch/reference/current/date_nanos.html[date nanoseconds]. +If the tracking field is of this data type, you can use an extra placeholder called `:present` to inject the nano-second based value of "now-30s". +This placeholder is useful as the right-hand side of a range filter, allowing the collection of +new data but leaving partially-searchable bulk request data to the next scheduled job. + +[id="{version}-plugins-{type}s-{plugin}-tracking-sample"] +===== Sample configuration: Track field value across runs + +This section contains a series of steps to help you set up the "tailing" of data being written to a set of indices, using a date nanosecond field added by an Elasticsearch ingest pipeline and the `tracking_field` capability of this plugin. + +. Create ingest pipeline that adds Elasticsearch's `_ingest.timestamp` field to the documents as `event.ingested`: ++ +[source, json] + PUT _ingest/pipeline/my-pipeline + { + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.putIfAbsent(\"event\", [:]); ctx.event.ingested = metadata().now.format(DateTimeFormatter.ISO_INSTANT);" + } + } + ] + } + +[start=2] +. Create an index mapping where the tracking field is of date nanosecond type and invokes the defined pipeline: ++ +[source, json] + PUT /_template/my_template + { + "index_patterns": ["test-*"], + "settings": { + "index.default_pipeline": "my-pipeline", + }, + "mappings": { + "properties": { + "event": { + "properties": { + "ingested": { + "type": "date_nanos", + "format": "strict_date_optional_time_nanos" + } + } + } + } + } + } + +[start=3] +. Define a query that looks at all data of the indices, sorted by the tracking field, and with a range filter since the last value seen until present: ++ +[source,json] +{ + "query": { + "range": { + "event.ingested": { + "gt": ":last_value", + "lt": ":present" + } + } + }, + "sort": [ + { + "event.ingested": { + "order": "asc", + "format": "strict_date_optional_time_nanos", + "numeric_type": "date_nanos" + } + } + ] +} + +[start=4] +. Configure the Elasticsearch input to query the indices with the query defined above, every minute, and track the `event.ingested` field: ++ +[source, ruby] + input { + elasticsearch { + id => tail_test_index + hosts => [ 'https://..'] + api_key => '....' + index => 'test-*' + query => '{ "query": { "range": { "event.ingested": { "gt": ":last_value", "lt": ":present"}}}, "sort": [ { "event.ingested": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" } } ] }' + tracking_field => "[event][ingested]" + slices => 5 # optional use of slices to speed data processing, should be equal to or less than number of primary shards + schedule => '* * * * *' # every minute + schedule_overlap => false # don't accumulate jobs if one takes longer than 1 minute + } + } + +With this sample setup, new documents are indexed into a `test-*` index. +The next scheduled run: + +* selects all new documents since the last observed value of the tracking field, +* uses {ref}/point-in-time-api.html#point-in-time-api[Point in time (PIT)] + {ref}/paginate-search-results.html#search-after[Search after] to paginate through all the data, and +* updates the value of the field at the end of the pagination. + +[id="{version}-plugins-{type}s-{plugin}-esql"] +==== {esql} support + +.Technical Preview +**** +The {esql} feature that allows using ES|QL queries with this plugin is in Technical Preview. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +{es} Query Language ({esql}) provides a SQL-like interface for querying your {es} data. + +To use {esql}, this plugin needs to be installed in {ls} 8.17.4 or newer, and must be connected to {es} 8.11 or newer. + +To configure {esql} query in the plugin, set the `query_type` to `esql` and provide your {esql} query in the `query` parameter. + +IMPORTANT: {esql} is evolving and may still have limitations with regard to result size or supported field types. We recommend understanding https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-limitations.html[ES|QL current limitations] before using it in production environments. + +The following is a basic scheduled {esql} query that runs hourly: +[source, ruby] + input { + elasticsearch { + id => hourly_cron_job + hosts => [ 'https://..'] + api_key => '....' + query_type => 'esql' + query => ' + FROM food-index + | WHERE spicy_level = "hot" AND @timestamp > NOW() - 1 hour + | LIMIT 500 + ' + schedule => '0 * * * *' # every hour at min 0 + } + } + +Set `config.support_escapes: true` in `logstash.yml` if you need to escape special chars in the query. + +NOTE: With {esql} query, {ls} doesn't generate `event.original`. + +[id="{version}-plugins-{type}s-{plugin}-esql-event-mapping"] +===== Mapping {esql} result to {ls} event +{esql} returns query results in a structured tabular format, where data is organized into _columns_ (fields) and _values_ (entries). +The plugin maps each value entry to an event, populating corresponding fields. +For example, a query might produce a table like: + +[cols="2,1,1,1,2",options="header"] +|=== +|`timestamp` |`user_id` | `action` | `status.code` | `status.desc` + +|2025-04-10T12:00:00 |123 |login |200 | Success +|2025-04-10T12:05:00 |456 |purchase |403 | Forbidden (unauthorized user) +|=== + +For this case, the plugin emits two events look like +[source, json] +[ + { + "timestamp": "2025-04-10T12:00:00", + "user_id": 123, + "action": "login", + "status": { + "code": 200, + "desc": "Success" + } + }, + { + "timestamp": "2025-04-10T12:05:00", + "user_id": 456, + "action": "purchase", + "status": { + "code": 403, + "desc": "Forbidden (unauthorized user)" + } + } +] + +NOTE: If your index has a mapping with sub-objects where `status.code` and `status.desc` actually dotted fields, they appear in {ls} events as a nested structure. + +[id="{version}-plugins-{type}s-{plugin}-esql-multifields"] +===== Conflict on multi-fields + +{esql} query fetches all parent and sub-fields fields if your {es} index has https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/multi-fields[multi-fields] or https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/subobjects[subobjects]. +Since {ls} events cannot contain parent field's concrete value and sub-field values together, the plugin ignores sub-fields with warning and includes parent. +We recommend using the `RENAME` (or `DROP` to avoid warnings) keyword in your {esql} query explicitly rename the fields to include sub-fields into the event. + +This a common occurrence if your template or mapping follows the pattern of always indexing strings as "text" (`field`) + " keyword" (`field.keyword`) multi-field. +In this case it's recommended to do `KEEP field` if the string is identical and there is only one subfield as the engine will optimize and retrieve the keyword, otherwise you can do `KEEP field.keyword | RENAME field.keyword as field`. + +To illustrate the situation with example, assuming your mapping has a time `time` field with `time.min` and `time.max` sub-fields as following: +[source, ruby] + "properties": { + "time": { "type": "long" }, + "time.min": { "type": "long" }, + "time.max": { "type": "long" } + } + +The {esql} result will contain all three fields but the plugin cannot map them into {ls} event. +To avoid this, you can use the `RENAME` keyword to rename the `time` parent field to get all three fields with unique fields. +[source, ruby] + ... + query => 'FROM my-index | RENAME time AS time.current' + ... + +For comprehensive {esql} syntax reference and best practices, see the https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-syntax.html[{esql} documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["dsl","esql"]`|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schedule_overlap>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The path to store the last observed value of the tracking field, when used. +By default this file is stored as `/plugins/inputs/elasticsearch//last_run_value`. + +This setting should point to file, not a directory, and Logstash must have read+write access to this file. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. +Accepted query shape is DSL or {esql} (when `query_type => 'esql'`). +Read the {ref}/query-dsl.html[{es} query DSL documentation] or {ref}/esql.html[{esql} documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-query_type"] +===== `query_type` + +* Value can be `dsl` or `esql` +* Default value is `dsl` + +Defines the <<{version}-plugins-{type}s-{plugin}-query>> shape. +When `dsl`, the query shape must be valid {es} JSON-style string. +When `esql`, the query shape must be a valid {esql} string and `index`, `size`, `slices`, `search_api`, `docinfo`, `docinfo_target`, `docinfo_fields`, `response_type` and `tracking_field` parameters are not allowed. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations`, `esql` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. + +The default `hits` will generate one event per returned document (i.e. "hit"). + +When set to `aggregations`, a single {ls} event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-schedule_overlap"] +===== `schedule_overlap` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to allow queuing of a scheduled run if a run is occurring. +While this is ideal for ensuring a new run happens immediately after the previous on finishes if there +is a lot of work to do, but given the queue is unbounded it may lead to an out of memory over long periods of time +if the queue grows continuously. + +When in doubt, set `schedule_overlap` to false (it may become the default value in the future). + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + +[id="{version}-plugins-{type}s-{plugin}-tracking_field"] +===== `tracking_field` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Which field from the last event of a previous run will be used a cursor value for the following run. +The value of this field is injected into each query if the query uses the placeholder `:last_value`. +For the first query after a pipeline is started, the value used is either read from <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> file, +or taken from <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> setting. + +Note: The tracking value is updated after each page is read and at the end of each Point in Time. In case of a crash the last saved value will be used so some duplication of data can occur. For this reason the use of unique document IDs for each event is recommended in the downstream destination. + +[id="{version}-plugins-{type}s-{plugin}-tracking_field_seed"] +===== `tracking_field_seed` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `"1970-01-01T00:00:00.000000000Z"` + +The starting value for the <<{version}-plugins-{type}s-{plugin}-tracking_field>> if there is no <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> already. +This field defaults to the nanosecond precision ISO8601 representation of `epoch`, or "1970-01-01T00:00:00.000000000Z", given nano-second precision timestamps are the +most reliable data format to use for this feature. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Input deprecated configuration options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +If enabled, SSL will be used when communicating with the Elasticsearch +server (i.e. HTTPS will be used instead of plain HTTP). + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[4.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +When certificate validation is disabled, this plugin implicitly trusts the machine +resolved at the given address without validating its proof-of-identity. +In this scenario, the plugin can transmit credentials to or process data from an untrustworthy +man-in-the-middle or other compromised infrastructure. +More information on the importance of certificate verification: +**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v5.0.0.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v5.0.0.asciidoc new file mode 100644 index 000000000..34a4e9f84 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v5.0.0.asciidoc @@ -0,0 +1,648 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.0.0 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v5.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +When not explicitly set, SSL will be automatically enabled if any of the specified hosts use HTTPS. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Input Obsolete Configuration Options + +WARNING: As of version `5.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v5.0.2.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v5.0.2.asciidoc new file mode 100644 index 000000000..798bb09e7 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v5.0.2.asciidoc @@ -0,0 +1,655 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.0.2 +:release_date: 2025-03-17 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v5.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<{version}-plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace. + - When <<{version}-plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<{version}-plugins-{type}s-{plugin}-target>> and <<{version}-plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +When not explicitly set, SSL will be automatically enabled if any of the specified hosts use HTTPS. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Input Obsolete Configuration Options + +WARNING: As of version `5.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v5.1.0.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v5.1.0.asciidoc new file mode 100644 index 000000000..ebff51091 --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v5.1.0.asciidoc @@ -0,0 +1,832 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.1.0 +:release_date: 2025-04-07 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v5.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + +[id="{version}-plugins-{type}s-{plugin}-scheduling"] +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<{version}-plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace. + - When <<{version}-plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<{version}-plugins-{type}s-{plugin}-target>> and <<{version}-plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict. + +[id="{version}-plugins-{type}s-{plugin}-cursor"] +==== Tracking a field's value across runs + +.Technical Preview: Tracking a field's value +**** +The feature that allows tracking a field's value across runs is in _Technical Preview_. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +Some uses cases require tracking the value of a particular field between two jobs. +Examples include: + +* avoiding the need to re-process the entire result set of a long query after an unplanned restart +* grabbing only new data from an index instead of processing the entire set on each job. + +The Elasticsearch input plugin provides the <<{version}-plugins-{type}s-{plugin}-tracking_field>> and <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> options. +When <<{version}-plugins-{type}s-{plugin}-tracking_field>> is set, the plugin records the value of that field for the last document retrieved in a run into +a file. +(The file location defaults to <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>>.) + +You can then inject this value in the query using the placeholder `:last_value`. +The value will be injected into the query before execution, and then updated after the query completes if new data was found. + +This feature works best when: + +* the query sorts by the tracking field, +* the timestamp field is added by {es}, and +* the field type has enough resolution so that two events are unlikely to have the same value. + +Consider using a tracking field whose type is https://www.elastic.co/guide/en/elasticsearch/reference/current/date_nanos.html[date nanoseconds]. +If the tracking field is of this data type, you can use an extra placeholder called `:present` to inject the nano-second based value of "now-30s". +This placeholder is useful as the right-hand side of a range filter, allowing the collection of +new data but leaving partially-searchable bulk request data to the next scheduled job. + +[id="{version}-plugins-{type}s-{plugin}-tracking-sample"] +===== Sample configuration: Track field value across runs + +This section contains a series of steps to help you set up the "tailing" of data being written to a set of indices, using a date nanosecond field added by an Elasticsearch ingest pipeline and the `tracking_field` capability of this plugin. + +. Create ingest pipeline that adds Elasticsearch's `_ingest.timestamp` field to the documents as `event.ingested`: ++ +[source, json] + PUT _ingest/pipeline/my-pipeline + { + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.putIfAbsent(\"event\", [:]); ctx.event.ingested = metadata().now.format(DateTimeFormatter.ISO_INSTANT);" + } + } + ] + } + +[start=2] +. Create an index mapping where the tracking field is of date nanosecond type and invokes the defined pipeline: ++ +[source, json] + PUT /_template/my_template + { + "index_patterns": ["test-*"], + "settings": { + "index.default_pipeline": "my-pipeline", + }, + "mappings": { + "properties": { + "event": { + "properties": { + "ingested": { + "type": "date_nanos", + "format": "strict_date_optional_time_nanos" + } + } + } + } + } + } + +[start=3] +. Define a query that looks at all data of the indices, sorted by the tracking field, and with a range filter since the last value seen until present: ++ +[source,json] +{ + "query": { + "range": { + "event.ingested": { + "gt": ":last_value", + "lt": ":present" + } + } + }, + "sort": [ + { + "event.ingested": { + "order": "asc", + "format": "strict_date_optional_time_nanos", + "numeric_type": "date_nanos" + } + } + ] +} + +[start=4] +. Configure the Elasticsearch input to query the indices with the query defined above, every minute, and track the `event.ingested` field: ++ +[source, ruby] + input { + elasticsearch { + id => tail_test_index + hosts => [ 'https://..'] + api_key => '....' + index => 'test-*' + query => '{ "query": { "range": { "event.ingested": { "gt": ":last_value", "lt": ":present"}}}, "sort": [ { "event.ingested": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" } } ] }' + tracking_field => "[event][ingested]" + slices => 5 # optional use of slices to speed data processing, should be equal to or less than number of primary shards + schedule => '* * * * *' # every minute + schedule_overlap => false # don't accumulate jobs if one takes longer than 1 minute + } + } + +With this sample setup, new documents are indexed into a `test-*` index. +The next scheduled run: + +* selects all new documents since the last observed value of the tracking field, +* uses {ref}/point-in-time-api.html#point-in-time-api[Point in time (PIT)] + {ref}/paginate-search-results.html#search-after[Search after] to paginate through all the data, and +* updates the value of the field at the end of the pagination. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schedule_overlap>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The path to store the last observed value of the tracking field, when used. +By default this file is stored as `/plugins/inputs/elasticsearch//last_run_value`. + +This setting should point to file, not a directory, and Logstash must have read+write access to this file. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL +documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. +The default `hits` will generate one event per returned document (i.e. "hit"). +When set to `aggregations`, a single Logstash event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-schedule_overlap"] +===== `schedule_overlap` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to allow queuing of a scheduled run if a run is occurring. +While this is ideal for ensuring a new run happens immediately after the previous on finishes if there +is a lot of work to do, but given the queue is unbounded it may lead to an out of memory over long periods of time +if the queue grows continuously. + +When in doubt, set `schedule_overlap` to false (it may become the default value in the future). + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +When not explicitly set, SSL will be automatically enabled if any of the specified hosts use HTTPS. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + +[id="{version}-plugins-{type}s-{plugin}-tracking_field"] +===== `tracking_field` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Which field from the last event of a previous run will be used a cursor value for the following run. +The value of this field is injected into each query if the query uses the placeholder `:last_value`. +For the first query after a pipeline is started, the value used is either read from <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> file, +or taken from <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> setting. + +Note: The tracking value is updated after each page is read and at the end of each Point in Time. In case of a crash the last saved value will be used so some duplication of data can occur. For this reason the use of unique document IDs for each event is recommended in the downstream destination. + +[id="{version}-plugins-{type}s-{plugin}-tracking_field_seed"] +===== `tracking_field_seed` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `"1970-01-01T00:00:00.000000000Z"` + +The starting value for the <<{version}-plugins-{type}s-{plugin}-tracking_field>> if there is no <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> already. +This field defaults to the nanosecond precision ISO8601 representation of `epoch`, or "1970-01-01T00:00:00.000000000Z", given nano-second precision timestamps are the +most reliable data format to use for this feature. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Input Obsolete Configuration Options + +WARNING: As of version `5.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/elasticsearch-v5.2.0.asciidoc b/docs/versioned-plugins/inputs/elasticsearch-v5.2.0.asciidoc new file mode 100644 index 000000000..fa249f82d --- /dev/null +++ b/docs/versioned-plugins/inputs/elasticsearch-v5.2.0.asciidoc @@ -0,0 +1,950 @@ +:plugin: elasticsearch +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.2.0 +:release_date: 2025-06-06 +:changelog_url: https://github.com/logstash-plugins/logstash-input-elasticsearch/blob/v5.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read from an Elasticsearch cluster, based on search query results. +This is useful for replaying test logs, reindexing, etc. +You can periodically schedule ingestion using a cron syntax +(see `schedule` setting) or run the query one time to load +data into Logstash. + +Example: +[source,ruby] + input { + # Read all documents from Elasticsearch matching the given query + elasticsearch { + hosts => "localhost" + query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }' + } + } + +This would create an Elasticsearch query with the following format: +[source,json] + curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{ + "query": { + "match": { + "statuscode": 200 + } + }, + "sort": [ "_doc" ] + }' + +[id="{version}-plugins-{type}s-{plugin}-scheduling"] +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin +uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information. + +The plugin logs a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + +[id="{version}-plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<{version}-plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace. + - When <<{version}-plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<{version}-plugins-{type}s-{plugin}-target>> and <<{version}-plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict. + +[id="{version}-plugins-{type}s-{plugin}-cursor"] +==== Tracking a field's value across runs + +.Technical Preview: Tracking a field's value +**** +The feature that allows tracking a field's value across runs is in _Technical Preview_. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +Some uses cases require tracking the value of a particular field between two jobs. +Examples include: + +* avoiding the need to re-process the entire result set of a long query after an unplanned restart +* grabbing only new data from an index instead of processing the entire set on each job. + +The Elasticsearch input plugin provides the <<{version}-plugins-{type}s-{plugin}-tracking_field>> and <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> options. +When <<{version}-plugins-{type}s-{plugin}-tracking_field>> is set, the plugin records the value of that field for the last document retrieved in a run into +a file. +(The file location defaults to <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>>.) + +You can then inject this value in the query using the placeholder `:last_value`. +The value will be injected into the query before execution, and then updated after the query completes if new data was found. + +This feature works best when: + +* the query sorts by the tracking field, +* the timestamp field is added by {es}, and +* the field type has enough resolution so that two events are unlikely to have the same value. + +Consider using a tracking field whose type is https://www.elastic.co/guide/en/elasticsearch/reference/current/date_nanos.html[date nanoseconds]. +If the tracking field is of this data type, you can use an extra placeholder called `:present` to inject the nano-second based value of "now-30s". +This placeholder is useful as the right-hand side of a range filter, allowing the collection of +new data but leaving partially-searchable bulk request data to the next scheduled job. + +[id="{version}-plugins-{type}s-{plugin}-tracking-sample"] +===== Sample configuration: Track field value across runs + +This section contains a series of steps to help you set up the "tailing" of data being written to a set of indices, using a date nanosecond field added by an Elasticsearch ingest pipeline and the `tracking_field` capability of this plugin. + +. Create ingest pipeline that adds Elasticsearch's `_ingest.timestamp` field to the documents as `event.ingested`: ++ +[source, json] + PUT _ingest/pipeline/my-pipeline + { + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.putIfAbsent(\"event\", [:]); ctx.event.ingested = metadata().now.format(DateTimeFormatter.ISO_INSTANT);" + } + } + ] + } + +[start=2] +. Create an index mapping where the tracking field is of date nanosecond type and invokes the defined pipeline: ++ +[source, json] + PUT /_template/my_template + { + "index_patterns": ["test-*"], + "settings": { + "index.default_pipeline": "my-pipeline", + }, + "mappings": { + "properties": { + "event": { + "properties": { + "ingested": { + "type": "date_nanos", + "format": "strict_date_optional_time_nanos" + } + } + } + } + } + } + +[start=3] +. Define a query that looks at all data of the indices, sorted by the tracking field, and with a range filter since the last value seen until present: ++ +[source,json] +{ + "query": { + "range": { + "event.ingested": { + "gt": ":last_value", + "lt": ":present" + } + } + }, + "sort": [ + { + "event.ingested": { + "order": "asc", + "format": "strict_date_optional_time_nanos", + "numeric_type": "date_nanos" + } + } + ] +} + +[start=4] +. Configure the Elasticsearch input to query the indices with the query defined above, every minute, and track the `event.ingested` field: ++ +[source, ruby] + input { + elasticsearch { + id => tail_test_index + hosts => [ 'https://..'] + api_key => '....' + index => 'test-*' + query => '{ "query": { "range": { "event.ingested": { "gt": ":last_value", "lt": ":present"}}}, "sort": [ { "event.ingested": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" } } ] }' + tracking_field => "[event][ingested]" + slices => 5 # optional use of slices to speed data processing, should be equal to or less than number of primary shards + schedule => '* * * * *' # every minute + schedule_overlap => false # don't accumulate jobs if one takes longer than 1 minute + } + } + +With this sample setup, new documents are indexed into a `test-*` index. +The next scheduled run: + +* selects all new documents since the last observed value of the tracking field, +* uses {ref}/point-in-time-api.html#point-in-time-api[Point in time (PIT)] + {ref}/paginate-search-results.html#search-after[Search after] to paginate through all the data, and +* updates the value of the field at the end of the pagination. + +[id="{version}-plugins-{type}s-{plugin}-esql"] +==== {esql} support + +.Technical Preview +**** +The {esql} feature that allows using ES|QL queries with this plugin is in Technical Preview. +Configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. +**** + +{es} Query Language ({esql}) provides a SQL-like interface for querying your {es} data. + +To use {esql}, this plugin needs to be installed in {ls} 8.17.4 or newer, and must be connected to {es} 8.11 or newer. + +To configure {esql} query in the plugin, set the `query_type` to `esql` and provide your {esql} query in the `query` parameter. + +IMPORTANT: {esql} is evolving and may still have limitations with regard to result size or supported field types. We recommend understanding https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-limitations.html[ES|QL current limitations] before using it in production environments. + +The following is a basic scheduled {esql} query that runs hourly: +[source, ruby] + input { + elasticsearch { + id => hourly_cron_job + hosts => [ 'https://..'] + api_key => '....' + query_type => 'esql' + query => ' + FROM food-index + | WHERE spicy_level = "hot" AND @timestamp > NOW() - 1 hour + | LIMIT 500 + ' + schedule => '0 * * * *' # every hour at min 0 + } + } + +Set `config.support_escapes: true` in `logstash.yml` if you need to escape special chars in the query. + +NOTE: With {esql} query, {ls} doesn't generate `event.original`. + +[id="{version}-plugins-{type}s-{plugin}-esql-event-mapping"] +===== Mapping {esql} result to {ls} event +{esql} returns query results in a structured tabular format, where data is organized into _columns_ (fields) and _values_ (entries). +The plugin maps each value entry to an event, populating corresponding fields. +For example, a query might produce a table like: + +[cols="2,1,1,1,2",options="header"] +|=== +|`timestamp` |`user_id` | `action` | `status.code` | `status.desc` + +|2025-04-10T12:00:00 |123 |login |200 | Success +|2025-04-10T12:05:00 |456 |purchase |403 | Forbidden (unauthorized user) +|=== + +For this case, the plugin emits two events look like +[source, json] +[ + { + "timestamp": "2025-04-10T12:00:00", + "user_id": 123, + "action": "login", + "status": { + "code": 200, + "desc": "Success" + } + }, + { + "timestamp": "2025-04-10T12:05:00", + "user_id": 456, + "action": "purchase", + "status": { + "code": 403, + "desc": "Forbidden (unauthorized user)" + } + } +] + +NOTE: If your index has a mapping with sub-objects where `status.code` and `status.desc` actually dotted fields, they appear in {ls} events as a nested structure. + +[id="{version}-plugins-{type}s-{plugin}-esql-multifields"] +===== Conflict on multi-fields + +{esql} query fetches all parent and sub-fields fields if your {es} index has https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/multi-fields[multi-fields] or https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/subobjects[subobjects]. +Since {ls} events cannot contain parent field's concrete value and sub-field values together, the plugin ignores sub-fields with warning and includes parent. +We recommend using the `RENAME` (or `DROP` to avoid warnings) keyword in your {esql} query explicitly rename the fields to include sub-fields into the event. + +This a common occurrence if your template or mapping follows the pattern of always indexing strings as "text" (`field`) + " keyword" (`field.keyword`) multi-field. +In this case it's recommended to do `KEEP field` if the string is identical and there is only one subfield as the engine will optimize and retrieve the keyword, otherwise you can do `KEEP field.keyword | RENAME field.keyword as field`. + +To illustrate the situation with example, assuming your mapping has a time `time` field with `time.min` and `time.max` sub-fields as following: +[source, ruby] + "properties": { + "time": { "type": "long" }, + "time.min": { "type": "long" }, + "time.max": { "type": "long" } + } + +The {esql} result will contain all three fields but the plugin cannot map them into {ls} event. +To avoid this, you can use the `RENAME` keyword to rename the `time` parent field to get all three fields with unique fields. +[source, ruby] + ... + query => 'FROM my-index | RENAME time AS time.current' + ... + +For comprehensive {esql} syntax reference and best practices, see the https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-syntax.html[{esql} documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Input configuration options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["dsl","esql"]`|No +| <<{version}-plugins-{type}s-{plugin}-response_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["hits","aggregations"]`|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schedule_overlap>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-scroll>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-search_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "search_after", "scroll"]`|No +| <<{version}-plugins-{type}s-{plugin}-size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-slices>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout_seconds>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> | {logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch +{ref}/security-api-create-api-key.html[Create +API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout_seconds"] +===== `connect_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch. +Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The headers will be used for any kind of request. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + +[id="{version}-plugins-{type}s-{plugin}-docinfo"] +===== `docinfo` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If set, include Elasticsearch document information such as index, type, and +the id in the event. + +It might be important to note, with regards to metadata, that if you're +ingesting documents with the intent to re-index them (or just update them) +that the `action` option in the elasticsearch output wants to know how to +handle those things. It can be dynamically assigned with a field +added to the metadata. + +Example +[source, ruby] + input { + elasticsearch { + hosts => "es.production.mysite.org" + index => "mydata-2018.09.*" + query => '{ "query": { "query_string": { "query": "*" } } }' + size => 500 + scroll => "5m" + docinfo => true + docinfo_target => "[@metadata][doc]" + } + } + output { + elasticsearch { + index => "copy-of-production.%{[@metadata][doc][_index]}" + document_type => "%{[@metadata][doc][_type]}" + document_id => "%{[@metadata][doc][_id]}" + } + } + +If set, you can use metadata information in the <<{version}-plugins-{type}s-{plugin}-add_field>> common option. + +Example +[source, ruby] + input { + elasticsearch { + docinfo => true + docinfo_target => "[@metadata][doc]" + add_field => { + identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}" + } + } + } + + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_index", "_type", "_id"]` + +If document metadata storage is requested by enabling the `docinfo` option, this +option lists the metadata fields to save in the current event. See +{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for +more information. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_target"] +===== `docinfo_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"@metadata"` + ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"` + +If document metadata storage is requested by enabling the `docinfo` option, +this option names the field under which to store the metadata fields as subfields. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: CSV data added at root level + ** `v1`,`v8`: Elastic Common Schema compliant behavior + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +List of one or more Elasticsearch hosts to use for querying. Each host +can be either IP, HOST, IP:port, or HOST:port. The port defaults to +9200. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash-*"` + +The index or alias to search. +Check out {ref}/api-conventions.html#api-multi-index[Multi Indices +documentation] in the Elasticsearch documentation for info on +referencing multiple indices. + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The path to store the last observed value of the tracking field, when used. +By default this file is stored as `/plugins/inputs/elasticsearch//last_run_value`. + +This setting should point to file, not a directory, and Logstash must have read+write access to this file. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password to use together with the username in the `user` option +when authenticating to the Elasticsearch server. If set to an empty +string authentication will be disabled. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, this is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `'{ "sort": [ "_doc" ] }'` + +The query to be executed. +Accepted query shape is DSL or {esql} (when `query_type => 'esql'`). +Read the {ref}/query-dsl.html[{es} query DSL documentation] or {ref}/esql.html[{esql} documentation] for more information. + +When <<{version}-plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`, +the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more. + +[id="{version}-plugins-{type}s-{plugin}-query_type"] +===== `query_type` + +* Value can be `dsl` or `esql` +* Default value is `dsl` + +Defines the <<{version}-plugins-{type}s-{plugin}-query>> shape. +When `dsl`, the query shape must be valid {es} JSON-style string. +When `esql`, the query shape must be a valid {esql} string and `index`, `size`, `slices`, `search_api`, `docinfo`, `docinfo_target`, `docinfo_fields`, `response_type` and `tracking_field` parameters are not allowed. + +[id="{version}-plugins-{type}s-{plugin}-response_type"] +===== `response_type` + + * Value can be any of: `hits`, `aggregations`, `esql` + * Default value is `hits` + +Which part of the result to transform into Logstash events when processing the +response from the query. + +The default `hits` will generate one event per returned document (i.e. "hit"). + +When set to `aggregations`, a single {ls} event will be generated with the +contents of the `aggregations` object of the query's response. In this case the +`hits` object will be ignored. The parameter `size` will be always be set to +0 regardless of the default or user-defined value set in this plugin. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_seconds"] +===== `request_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, for a single request to Elasticsearch. +Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload +documents and/or the <<{version}-plugins-{type}s-{plugin}-size>> has been specified as a large value. + + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` + +The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message. +The default is 0 (no retry). This value should be equal to or greater than zero. + +NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-schedule_overlap"] +===== `schedule_overlap` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to allow queuing of a scheduled run if a run is occurring. +While this is ideal for ensuring a new run happens immediately after the previous on finishes if there +is a lot of work to do, but given the queue is unbounded it may lead to an out of memory over long periods of time +if the queue grows continuously. + +When in doubt, set `schedule_overlap` to false (it may become the default value in the future). + +[id="{version}-plugins-{type}s-{plugin}-scroll"] +===== `scroll` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +This parameter controls the keepalive time in seconds of the scrolling +request and initiates the scrolling process. The timeout applies per +round trip (i.e. between the previous scroll request, to the next). + +[id="{version}-plugins-{type}s-{plugin}-search_api"] +===== `search_api` + +* Value can be any of: `auto`, `search_after`, `scroll` +* Default value is `auto` + +With `auto` the plugin uses the `search_after` parameter for Elasticsearch version `8.0.0` or higher, otherwise the `scroll` API is used instead. + +`search_after` uses {ref}/point-in-time-api.html#point-in-time-api[point in time] and sort value to search. +The query requires at least one `sort` field, as described in the <<{version}-plugins-{type}s-{plugin}-query>> parameter. + +`scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended. + +[id="{version}-plugins-{type}s-{plugin}-size"] +===== `size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +This allows you to set the maximum number of hits returned per scroll. + +[id="{version}-plugins-{type}s-{plugin}-slices"] +===== `slices` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value. + * Sensible values range from 2 to about 8. + +In some cases, it is possible to improve overall throughput by consuming multiple +distinct slices of a query simultaneously using +{ref}/paginate-search-results.html#slice-scroll[sliced scrolls], +especially if the pipeline is spending significant time waiting on Elasticsearch +to provide results. + +If set, the `slices` parameter tells the plugin how many slices to divide the work +into, and will produce events from the slices in parallel until all of them are done +scrolling. + +NOTE: The Elasticsearch manual indicates that there can be _negative_ performance + implications to both the query and the Elasticsearch cluster when a scrolling + query uses more slices than shards in the index. + +If the `slices` parameter is left unset, the plugin will _not_ inject slice +instructions into the query. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `.cer` or `.pem` files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +When not explicitly set, SSL will be automatically enabled if any of the specified hosts use HTTPS. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout_seconds"] +===== `socket_timeout_seconds` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended. +Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query. + + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each hit's `_source` at the root level. +When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. +It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`). + +[id="{version}-plugins-{type}s-{plugin}-tracking_field"] +===== `tracking_field` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Which field from the last event of a previous run will be used a cursor value for the following run. +The value of this field is injected into each query if the query uses the placeholder `:last_value`. +For the first query after a pipeline is started, the value used is either read from <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> file, +or taken from <<{version}-plugins-{type}s-{plugin}-tracking_field_seed>> setting. + +Note: The tracking value is updated after each page is read and at the end of each Point in Time. In case of a crash the last saved value will be used so some duplication of data can occur. For this reason the use of unique document IDs for each event is recommended in the downstream destination. + +[id="{version}-plugins-{type}s-{plugin}-tracking_field_seed"] +===== `tracking_field_seed` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `"1970-01-01T00:00:00.000000000Z"` + +The starting value for the <<{version}-plugins-{type}s-{plugin}-tracking_field>> if there is no <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> already. +This field defaults to the nanosecond precision ISO8601 representation of `epoch`, or "1970-01-01T00:00:00.000000000Z", given nano-second precision timestamps are the +most reliable data format to use for this feature. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The username to use together with the password in the `password` +option when authenticating to the Elasticsearch server. If set to an +empty string authentication will be disabled. + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Input Obsolete Configuration Options + +WARNING: As of version `5.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ca_file | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/file-index.asciidoc b/docs/versioned-plugins/inputs/file-index.asciidoc index d78f5c9ca..9c80ccd96 100644 --- a/docs/versioned-plugins/inputs/file-index.asciidoc +++ b/docs/versioned-plugins/inputs/file-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2023-12-13 | <> | 2023-10-18 | <> | 2022-09-20 | <> | 2022-06-06 @@ -42,6 +43,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::file-v4.4.6.asciidoc[] include::file-v4.4.5.asciidoc[] include::file-v4.4.4.asciidoc[] include::file-v4.4.3.asciidoc[] diff --git a/docs/versioned-plugins/inputs/file-v4.4.6.asciidoc b/docs/versioned-plugins/inputs/file-v4.4.6.asciidoc new file mode 100644 index 000000000..fce2da1d6 --- /dev/null +++ b/docs/versioned-plugins/inputs/file-v4.4.6.asciidoc @@ -0,0 +1,536 @@ +:plugin: file +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.4.6 +:release_date: 2023-12-13 +:changelog_url: https://github.com/logstash-plugins/logstash-input-file/blob/v4.4.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== File input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Stream events from files, normally by tailing them in a manner +similar to `tail -0F` but optionally reading them from the +beginning. + +Normally, logging will add a newline to the end of each line written. +By default, each event is assumed to be one line +and a line is taken to be the text before a newline character. +If you would like to join multiple log lines into one event, +you'll want to use the multiline codec. +The plugin loops between discovering new files and processing +each discovered file. Discovered files have a lifecycle, they start off +in the "watched" or "ignored" state. Other states in the lifecycle are: +"active", "closed" and "unwatched" + +By default, a window of 4095 files is used to limit the number of file handles in use. +The processing phase has a number of stages: + +* Checks whether "closed" or "ignored" files have changed in size since last time and +if so puts them in the "watched" state. +* Selects enough "watched" files to fill the available space in the window, these files +are made "active". +* The active files are opened and read, each file is read from the last known position +to the end of current content (EOF) by default. + +In some cases it is useful to be able to control which files are read first, sorting, +and whether files are read completely or banded/striped. +Complete reading is *all of* file A then file B then file C and so on. +Banded or striped reading is *some of* file A then file B then file C and so on looping around +to file A again until all files are read. Banded reading is specified by changing +<<{version}-plugins-{type}s-{plugin}-file_chunk_count>> and perhaps <<{version}-plugins-{type}s-{plugin}-file_chunk_size>>. +Banding and sorting may be useful if you want some events from all files to appear +in Kibana as early as possible. + +The plugin has two modes of operation, Tail mode and Read mode. + +===== Tail mode + +In this mode the plugin aims to track changing files and emit new content as it's +appended to each file. In this mode, files are seen as a never ending stream of +content and EOF has no special significance. The plugin always assumes that +there will be more content. When files are rotated, the smaller or zero size is +detected, the current position is reset to zero and streaming continues. +A delimiter must be seen before the accumulated characters can be emitted as a line. + +===== Read mode + +In this mode the plugin treats each file as if it is content complete, that is, +a finite stream of lines and now EOF is significant. A last delimiter is not +needed because EOF means that the accumulated characters can be emitted as a line. +Further, EOF here means that the file can be closed and put in the "unwatched" +state - this automatically frees up space in the active window. This mode also +makes it possible to process compressed files as they are content complete. +Read mode also allows for an action to take place after processing the file completely. + +In the past attempts to simulate a Read mode while still assuming infinite streams +was not ideal and a dedicated Read mode is an improvement. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin adds metadata about event's source, and can be configured to do so +in an https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[ECS-compatible] way with <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +This metadata is added after the event has been decoded by the appropriate codec, +and will never overwrite existing values. + +|======== +| ECS Disabled | ECS `v1`, `v8` | Description + +| `host` | `[host][name]` | The name of the {ls} host that processed the event +| `path` | `[log][file][path]` | The full path to the log file from which the event originates +|======== + +==== Tracking of current position in watched files + +The plugin keeps track of the current position in each file by +recording it in a separate file named sincedb. This makes it +possible to stop and restart Logstash and have it pick up where it +left off without missing the lines that were added to the file while +Logstash was stopped. + +By default, the sincedb file is placed in the data directory of Logstash +with a filename based on the filename patterns being watched (i.e. the `path` option). +Thus, changing the filename patterns will result in a new sincedb file being used and +any existing current position state will be lost. If you change your patterns +with any frequency it might make sense to explicitly choose a sincedb path +with the `sincedb_path` option. + +A different `sincedb_path` must be used for each input. Using the same +path will cause issues. The read checkpoints for each input must be +stored in a different path so the information does not override. + +Files are tracked via an identifier. This identifier is made up of the +inode, major device number and minor device number. In windows, a different +identifier is taken from a `kernel32` API call. + +Sincedb records can now be expired meaning that read positions of older files +will not be remembered after a certain time period. File systems may need to reuse +inodes for new content. Ideally, we would not use the read position of old content, +but we have no reliable way to detect that inode reuse has occurred. This is more +relevant to Read mode where a great many files are tracked in the sincedb. +Bear in mind though, if a record has expired, a previously seen file will be read again. + +Sincedb files are text files with four (< v5.0.0), five or six columns: + +. The inode number (or equivalent). +. The major device number of the file system (or equivalent). +. The minor device number of the file system (or equivalent). +. The current byte offset within the file. +. The last active timestamp (a floating point number) +. The last known path that this record was matched to (for +old sincedb records converted to the new format, this is blank. + +On non-Windows systems you can obtain the inode number of a file +with e.g. `ls -li`. + +==== Reading from remote network volumes + +The file input is not thoroughly tested on remote filesystems such as NFS, +Samba, s3fs-fuse, etc, however NFS is occasionally tested. The file size as given by +the remote FS client is used to govern how much data to read at any given time to +prevent reading into allocated but yet unfilled memory. +As we use the device major and minor in the identifier to track "last read" +positions of files and on remount the device major and minor can change, the +sincedb records may not match across remounts. +Read mode might not be suitable for remote filesystems as the file size at +discovery on the client side may not be the same as the file size on the remote side +due to latency in the remote to client copy process. + +==== File rotation in Tail mode + +File rotation is detected and handled by this input, regardless of +whether the file is rotated via a rename or a copy operation. To +support programs that write to the rotated file for some time after +the rotation has taken place, include both the original filename and +the rotated filename (e.g. /var/log/syslog and /var/log/syslog.1) in +the filename patterns to watch (the `path` option). +For a rename, the inode will be detected as having moved from +`/var/log/syslog` to `/var/log/syslog.1` and so the "state" is moved +internally too, the old content will not be reread but any new content +on the renamed file will be read. +For copy/truncate the copied content into a new file path, if discovered, will +be treated as a new discovery and be read from the beginning. The copied file +paths should therefore not be in the filename patterns to watch (the `path` option). +The truncation will be detected and the "last read" position updated to zero. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== File Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[NOTE] +Duration settings can be specified in text form e.g. "250 ms", this string will be converted into +decimal seconds. There are quite a few supported natural and abbreviated durations, +see <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>> for the details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-check_archive_validity>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-close_older>> |{logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>>|No +| <<{version}-plugins-{type}s-{plugin}-delimiter>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-discover_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exclude>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-exit_after_read>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-file_chunk_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-file_chunk_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-file_completed_action>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["delete", "log", "log_and_delete"]`|No +| <<{version}-plugins-{type}s-{plugin}-file_completed_log_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-file_sort_by>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["last_modified", "path"]`|No +| <<{version}-plugins-{type}s-{plugin}-file_sort_direction>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["asc", "desc"]`|No +| <<{version}-plugins-{type}s-{plugin}-ignore_older>> |{logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>>|No +| <<{version}-plugins-{type}s-{plugin}-max_open_files>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["tail", "read"]`|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes +| <<{version}-plugins-{type}s-{plugin}-sincedb_clean_after>> |{logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>>|No +| <<{version}-plugins-{type}s-{plugin}-sincedb_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sincedb_write_interval>> |{logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>>|No +| <<{version}-plugins-{type}s-{plugin}-start_position>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["beginning", "end"]`|No +| <<{version}-plugins-{type}s-{plugin}-stat_interval>> |{logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>>|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-check_archive_validity"] +===== `check_archive_validity` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * The default is `false`. + +When set to `true`, this setting verifies that a compressed file is valid before +processing it. There are two passes through the file--one pass to +verify that the file is valid, and another pass to process the file. + +Validating a compressed file requires more processing time, but can prevent a +corrupt archive from causing looping. + + +[id="{version}-plugins-{type}s-{plugin}-close_older"] +===== `close_older` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>> + * Default value is `"1 hour"` + +The file input closes any files that were last read the specified +duration (seconds if a number is specified) ago. +This has different implications depending on if a file is being tailed or +read. If tailing, and there is a large time gap in incoming data the file +can be closed (allowing other files to be opened) but will be queued for +reopening when new data is detected. If reading, the file will be closed +after closed_older seconds from when the last bytes were read. +This setting is retained for backward compatibility if you upgrade the +plugin to 4.1.0+, are reading not tailing and do not switch to using Read mode. + +[id="{version}-plugins-{type}s-{plugin}-delimiter"] +===== `delimiter` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\n"` + +set the new line delimiter, defaults to "\n". Note that when reading compressed files +this setting is not used, instead the standard Windows or Unix line endings are used. + +[id="{version}-plugins-{type}s-{plugin}-discover_interval"] +===== `discover_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15` + +How often we expand the filename patterns in the `path` option to discover new files to watch. +This value is a multiple to `stat_interval`, e.g. if `stat_interval` is "500 ms" then new files +files could be discovered every 15 X 500 milliseconds - 7.5 seconds. +In practice, this will be the best case because the time taken to read new content needs to be factored in. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: sets non-ECS metadata on event (such as top-level `host`, `path`) +** `v1`,`v8`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`) +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the +https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-exclude"] +===== `exclude` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +Exclusions (matched against the filename, not full path). Filename +patterns are valid here, too. For example, if you have +[source,ruby] + path => "/var/log/*" + +In Tail mode, you might want to exclude gzipped files: +[source,ruby] + exclude => "*.gz" + +[id="{version}-plugins-{type}s-{plugin}-exit_after_read"] +===== `exit_after_read` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This option can be used in `read` mode to enforce closing all watchers when file gets read. +Can be used in situation when content of the file is static and won't change during execution. +When set to `true` it also disables active discovery of the files - only files that were in +the directories when process was started will be read. +It supports `sincedb` entries. When file was processed once, then modified - next run will only +read newly added entries. + +[id="{version}-plugins-{type}s-{plugin}-file_chunk_count"] +===== `file_chunk_count` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4611686018427387903` + +When combined with the `file_chunk_size`, this option sets how many chunks (bands or stripes) +are read from each file before moving to the next active file. +For example, a `file_chunk_count` of 32 and a `file_chunk_size` 32KB will process the next 1MB from each active file. +As the default is very large, the file is effectively read to EOF before moving to the next active file. + +[id="{version}-plugins-{type}s-{plugin}-file_chunk_size"] +===== `file_chunk_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB) + +File content is read off disk in blocks or chunks and lines are extracted from the chunk. +See <<{version}-plugins-{type}s-{plugin}-file_chunk_count>> to see why and when to change this setting +from the default. + +[id="{version}-plugins-{type}s-{plugin}-file_completed_action"] +===== `file_completed_action` + + * Value can be any of: `delete`, `log`, `log_and_delete` + * The default is `delete`. + +When in `read` mode, what action should be carried out when a file is done with. +If 'delete' is specified then the file will be deleted. If 'log' is specified +then the full path of the file is logged to the file specified in the +`file_completed_log_path` setting. If `log_and_delete` is specified then +both above actions take place. + +[id="{version}-plugins-{type}s-{plugin}-file_completed_log_path"] +===== `file_completed_log_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Which file should the completely read file paths be appended to. Only specify +this path to a file when `file_completed_action` is 'log' or 'log_and_delete'. +IMPORTANT: this file is appended to only - it could become very large. You are +responsible for file rotation. + +[id="{version}-plugins-{type}s-{plugin}-file_sort_by"] +===== `file_sort_by` + + * Value can be any of: `last_modified`, `path` + * The default is `last_modified`. + +Which attribute of a "watched" file should be used to sort them by. +Files can be sorted by modified date or full path alphabetic. +Previously the processing order of the discovered and therefore +"watched" files was OS dependent. + +[id="{version}-plugins-{type}s-{plugin}-file_sort_direction"] +===== `file_sort_direction` + + * Value can be any of: `asc`, `desc` + * The default is `asc`. + +Select between ascending and descending order when sorting "watched" files. +If oldest data first is important then the defaults of `last_modified` + `asc` are good. +If newest data first is more important then opt for `last_modified` + `desc`. +If you use special naming conventions for the file full paths then perhaps +`path` + `asc` will help to control the order of file processing. + +[id="{version}-plugins-{type}s-{plugin}-ignore_older"] +===== `ignore_older` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>> + * There is no default value for this setting. + +When the file input discovers a file that was last modified +before the specified duration (seconds if a number is specified), the file is ignored. +After it's discovery, if an ignored file is modified it is no +longer ignored and any new data is read. By default, this option is +disabled. Note this unit is in seconds. + +[id="{version}-plugins-{type}s-{plugin}-max_open_files"] +===== `max_open_files` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +What is the maximum number of file_handles that this input consumes +at any one time. Use close_older to close some files if you need to +process more files than this number. This should not be set to the +maximum the OS can do because file handles are needed for other +LS plugins and OS processes. +A default of 4095 is set in internally. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be either `tail` or `read`. + * The default value is `tail`. + +What mode do you want the file input to operate in. Tail a few files or +read many content-complete files. Read mode now supports gzip file processing. + +If `read` is specified, these settings can be used: + +* `ignore_older` (older files are not processed) +* `file_completed_action` (what action should be taken when the file is processed) +* `file_completed_log_path` (which file should the completed file path be logged to) + +If `read` is specified, these settings are ignored: + +* `start_position` (files are always read from the beginning) +* `close_older` (files are automatically 'closed' when EOF is reached) + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +The path(s) to the file(s) to use as an input. +You can use filename patterns here, such as `/var/log/*.log`. +If you use a pattern like `/var/log/**/*.log`, a recursive search +of `/var/log` will be done for all `*.log` files. +Paths must be absolute and cannot be relative. + +You may also configure multiple paths. See an example +on the https://www.elastic.co/guide/en/logstash/{branch}/configuration-file-structure.html#array[Logstash configuration page]. + +[id="{version}-plugins-{type}s-{plugin}-sincedb_clean_after"] +===== `sincedb_clean_after` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>> + * The default value for this setting is "2 weeks". + * If a number is specified then it is interpreted as *days* and can be decimal e.g. 0.5 is 12 hours. + +The sincedb record now has a last active timestamp associated with it. +If no changes are detected in a tracked file in the last N days its sincedb +tracking record expires and will not be persisted. +This option helps protect against the inode recycling problem. +Filebeat has an https://www.elastic.co/guide/en/beats/filebeat/{branch}/inode-reuse-issue.html[FAQ about inode recycling]. + +[id="{version}-plugins-{type}s-{plugin}-sincedb_path"] +===== `sincedb_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path of the sincedb database file (keeps track of the current +position of monitored log files) that will be written to disk. +The default will write sincedb files to `/plugins/inputs/file` +NOTE: it must be a file path and not a directory path + +[id="{version}-plugins-{type}s-{plugin}-sincedb_write_interval"] +===== `sincedb_write_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>> + * Default value is `"15 seconds"` + +How often (in seconds) to write a since database with the current position of +monitored log files. + +[id="{version}-plugins-{type}s-{plugin}-start_position"] +===== `start_position` + + * Value can be any of: `beginning`, `end` + * Default value is `"end"` + +Choose where Logstash starts initially reading files: at the beginning or +at the end. The default behavior treats files like live streams and thus +starts at the end. If you have old data you want to import, set this +to 'beginning'. + +This option only modifies "first contact" situations where a file +is new and not seen before, i.e. files that don't have a current +position recorded in a sincedb file read by Logstash. If a file +has already been seen before, this option has no effect and the +position recorded in the sincedb file will be used. + +[id="{version}-plugins-{type}s-{plugin}-stat_interval"] +===== `stat_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] or <<{version}-plugins-{type}s-{plugin}-string_duration,string_duration>> + * Default value is `"1 second"` + +How often (in seconds) we stat files to see if they have been modified. +Increasing this interval will decrease the number of system calls we make, +but increase the time to detect new log lines. +[NOTE] +Discovering new files and checking whether they have grown/or shrunk occurs in a loop. +This loop will sleep for `stat_interval` seconds before looping again. However, if files +have grown, the new content is read and lines are enqueued. +Reading and enqueuing across all grown files can take time, especially if +the pipeline is congested. So the overall loop time is a combination of the +`stat_interval` and the file read time. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + + +[id="{version}-plugins-{type}s-{plugin}-string_duration"] +// Move this to the includes when we make string durations available generally. +==== String Durations + +Format is `number` `string` and the space between these is optional. +So "45s" and "45 s" are both valid. +[TIP] +Use the most suitable duration, for example, "3 days" rather than "72 hours". + +===== Weeks +Supported values: `w` `week` `weeks`, e.g. "2 w", "1 week", "4 weeks". + +===== Days +Supported values: `d` `day` `days`, e.g. "2 d", "1 day", "2.5 days". + +===== Hours +Supported values: `h` `hour` `hours`, e.g. "4 h", "1 hour", "0.5 hours". + +===== Minutes +Supported values: `m` `min` `minute` `minutes`, e.g. "45 m", "35 min", "1 minute", "6 minutes". + +===== Seconds +Supported values: `s` `sec` `second` `seconds`, e.g. "45 s", "15 sec", "1 second", "2.5 seconds". + +===== Milliseconds +Supported values: `ms` `msec` `msecs`, e.g. "500 ms", "750 msec", "50 msecs +[NOTE] +`milli` `millis` and `milliseconds` are not supported + +===== Microseconds +Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs" +[NOTE] +`micro` `micros` and `microseconds` are not supported + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/google_pubsub-index.asciidoc b/docs/versioned-plugins/inputs/google_pubsub-index.asciidoc index 389680bf0..b0a55b370 100644 --- a/docs/versioned-plugins/inputs/google_pubsub-index.asciidoc +++ b/docs/versioned-plugins/inputs/google_pubsub-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-10-15 | <> | 2023-08-22 | <> | 2022-09-13 | <> | 2018-08-17 @@ -17,6 +18,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::google_pubsub-v1.4.0.asciidoc[] include::google_pubsub-v1.3.0.asciidoc[] include::google_pubsub-v1.2.2.asciidoc[] include::google_pubsub-v1.2.1.asciidoc[] diff --git a/docs/versioned-plugins/inputs/google_pubsub-v1.4.0.asciidoc b/docs/versioned-plugins/inputs/google_pubsub-v1.4.0.asciidoc new file mode 100644 index 000000000..2cc8d45ab --- /dev/null +++ b/docs/versioned-plugins/inputs/google_pubsub-v1.4.0.asciidoc @@ -0,0 +1,275 @@ +:plugin: google_pubsub +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.4.0 +:release_date: 2024-10-15 +:changelog_url: https://github.com/logstash-plugins/logstash-input-google_pubsub/blob/v1.4.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Google_pubsub input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Author: Eric Johnson +Date: 2016-06-01 + +Copyright 2016 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Google deps +This is a https://github.com/elastic/logstash[Logstash] input plugin for +https://cloud.google.com/pubsub/[Google Pub/Sub]. The plugin can subscribe +to a topic and ingest messages. + +The main motivation behind the development of this plugin was to ingest +https://cloud.google.com/logging/[Stackdriver Logging] messages via the +https://cloud.google.com/logging/docs/export/using_exported_logs[Exported Logs] +feature of Stackdriver Logging. + +==== Prerequisites + +You must first create a Google Cloud Platform project and enable the +Google Pub/Sub API. If you intend to use the plugin ingest Stackdriver Logging +messages, you must also enable the Stackdriver Logging API and configure log +exporting to Pub/Sub. There is plentiful information on +https://cloud.google.com/ to get started: + +- Google Cloud Platform Projects and https://cloud.google.com/docs/overview/[Overview] +- Google Cloud Pub/Sub https://cloud.google.com/pubsub/[documentation] +- Stackdriver Logging https://cloud.google.com/logging/[documentation] + +==== Cloud Pub/Sub + +Currently, this module requires you to create a `topic` manually and specify +it in the logstash config file. You must also specify a `subscription`, but +the plugin will attempt to create the pull-based `subscription` on its own. + +All messages received from Pub/Sub will be converted to a logstash `event` +and added to the processing pipeline queue. All Pub/Sub messages will be +`acknowledged` and removed from the Pub/Sub `topic` (please see more about +https://cloud.google.com/pubsub/overview#concepts)[Pub/Sub concepts]. + +It is generally assumed that incoming messages will be in JSON and added to +the logstash `event` as-is. However, if a plain text message is received, the +plugin will return the raw text in as `raw_message` in the logstash `event`. + +==== Authentication + +You have two options for authentication depending on where you run Logstash. + +1. If you are running Logstash outside of Google Cloud Platform, then you will +path to the JSON private key file in your config in `json_key_file` setting. +If you don't want to upload the file and use its content as a string, then add the content +of JSON private key in `json_key_file_content` setting. You must assign sufficient +roles to the Service Account to create a subscription and to pull messages +from the subscription. Learn more about GCP Service Accounts and IAM roles +here: + + - Google Cloud Platform IAM https://cloud.google.com/iam/[overview] + - Creating Service Accounts https://cloud.google.com/iam/docs/creating-managing-service-accounts[overview] + - Granting Roles https://cloud.google.com/iam/docs/granting-roles-to-service-accounts[overview] + +2. If you are running Logstash on a Google Compute Engine instance, you may opt +to use Application Default Credentials. In this case, you will not need to +specify a JSON private key file in your config. + +==== Stackdriver Logging (optional) + +If you intend to use the logstash plugin for Stackdriver Logging message +ingestion, you must first manually set up the Export option to Cloud Pub/Sub and +the manually create the `topic`. Please see the more detailed instructions at, +https://cloud.google.com/logging/docs/export/using_exported_logs [Exported Logs] +and ensure that the https://cloud.google.com/logging/docs/export/configure_export#manual-access-pubsub[necessary permissions] +have also been manually configured. + +Logging messages from Stackdriver Logging exported to Pub/Sub are received as +JSON and converted to a logstash `event` as-is in +https://cloud.google.com/logging/docs/export/using_exported_logs#log_entries_in_google_pubsub_topics[this format]. + +==== Sample Configuration + +Below is a copy of the included `example.conf-tmpl` file that shows a basic +configuration for this plugin. + +[source,ruby] +---------------------------------- +input { + google_pubsub { + # Your GCP project id (name) + project_id => "my-project-1234" + + # The topic name below is currently hard-coded in the plugin. You + # must first create this topic by hand and ensure you are exporting + # logging to this pubsub topic. + topic => "logstash-input-dev" + + # The subscription name is customizeable. The plugin will attempt to + # create the subscription (but use the hard-coded topic name above). + subscription => "logstash-sub" + + # If you are running logstash within GCE, it will use + # Application Default Credentials and use GCE's metadata + # service to fetch tokens. However, if you are running logstash + # outside of GCE, you will need to specify the service account's + # JSON key file below. + #json_key_file => "/home/erjohnso/pkey.json" + + # Should the plugin attempt to create the subscription on startup? + # This is not recommended for security reasons but may be useful in + # some cases. + #create_subscription => false + } +} +output { stdout { codec => rubydebug } } +---------------------------------- + +==== Metadata and Attributes + +The original Pub/Sub message is preserved in the special Logstash +`[@metadata][pubsub_message]` field so you can fetch: + +* Message attributes +* The origiginal base64 data +* Pub/Sub message ID for de-duplication +* Publish time + +You MUST extract any fields you want in a filter prior to the data being sent +to an output because Logstash deletes `@metadata` fields otherwise. + +See the PubsubMessage +https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage[documentation] +for a full description of the fields. + +Example to get the message ID: + +[source,ruby] +---------------------------------- +input {google_pubsub {...}} + +filter { + mutate { + add_field => { "messageId" => "%{[@metadata][pubsub_message][messageId]}" } + } +} + +output {...} +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Google_pubsub Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-json_key_file>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-max_messages>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-project_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-subscription>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-topic>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-include_metadata>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-create_subscription>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-json_key_file"] +===== `json_key_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If logstash is running within Google Compute Engine, the plugin will use +GCE's Application Default Credentials. Outside of GCE, you will need to +specify a Service Account JSON key file. + +[id="{version}-plugins-{type}s-{plugin}-max_messages"] +===== `max_messages` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +The maximum number of messages returned per request. +The Pub/Sub system may return fewer than the number specified. + +[id="{version}-plugins-{type}s-{plugin}-project_id"] +===== `project_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Google Cloud Project ID (name, not number). + +[id="{version}-plugins-{type}s-{plugin}-subscription"] +===== `subscription` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + + + +[id="{version}-plugins-{type}s-{plugin}-topic"] +===== `topic` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Google Cloud Pub/Sub Topic and Subscription. +Note that the topic must be created manually with Cloud Logging +pre-configured export to PubSub configured to use the defined topic. +The subscription will be created automatically by the plugin. + +[id="{version}-plugins-{type}s-{plugin}-include_metadata"] +===== `include_metadata` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false`. + +If set true, will include the full message data in the `[@metadata][pubsub_message]` field. + +[id="{version}-plugins-{type}s-{plugin}-create_subscription"] +===== `create_subscription` + +added[1.2.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false`. + +If true, the plugin will try to create the subscription before publishing. +Note: this requires additional permissions to be granted to the client and is _not_ +recommended for most use-cases. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-index.asciidoc b/docs/versioned-plugins/inputs/http-index.asciidoc index eb94a5d56..9dc1580b2 100644 --- a/docs/versioned-plugins/inputs/http-index.asciidoc +++ b/docs/versioned-plugins/inputs/http-index.asciidoc @@ -5,6 +5,19 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-09-04 +| <> | 2025-02-12 +| <> | 2025-02-04 +| <> | 2024-12-19 +| <> | 2024-12-18 +| <> | 2025-09-04 +| <> | 2025-02-12 +| <> | 2025-02-04 +| <> | 2024-12-19 +| <> | 2024-11-18 +| <> | 2024-10-18 +| <> | 2024-08-28 +| <> | 2024-04-23 | <> | 2023-11-28 | <> | 2023-10-18 | <> | 2023-06-29 @@ -42,6 +55,19 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::http-v4.1.3.asciidoc[] +include::http-v4.1.2.asciidoc[] +include::http-v4.1.1.asciidoc[] +include::http-v4.1.0.asciidoc[] +include::http-v4.0.0.asciidoc[] +include::http-v3.10.3.asciidoc[] +include::http-v3.10.2.asciidoc[] +include::http-v3.10.1.asciidoc[] +include::http-v3.10.0.asciidoc[] +include::http-v3.9.2.asciidoc[] +include::http-v3.9.1.asciidoc[] +include::http-v3.9.0.asciidoc[] +include::http-v3.8.1.asciidoc[] include::http-v3.8.0.asciidoc[] include::http-v3.7.3.asciidoc[] include::http-v3.7.2.asciidoc[] diff --git a/docs/versioned-plugins/inputs/http-v3.10.0.asciidoc b/docs/versioned-plugins/inputs/http-v3.10.0.asciidoc new file mode 100644 index 000000000..25ce34775 --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.10.0.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.10.0 +:release_date: 2024-12-19 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.10.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v3.10.1.asciidoc b/docs/versioned-plugins/inputs/http-v3.10.1.asciidoc new file mode 100644 index 000000000..282d7e50f --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.10.1.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.10.1 +:release_date: 2025-02-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.10.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v3.10.2.asciidoc b/docs/versioned-plugins/inputs/http-v3.10.2.asciidoc new file mode 100644 index 000000000..bc54f174b --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.10.2.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.10.2 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.10.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v3.10.3.asciidoc b/docs/versioned-plugins/inputs/http-v3.10.3.asciidoc new file mode 100644 index 000000000..867a82c56 --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.10.3.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.10.3 +:release_date: 2025-09-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.10.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v3.8.1.asciidoc b/docs/versioned-plugins/inputs/http-v3.8.1.asciidoc new file mode 100644 index 000000000..0b157c6ba --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.8.1.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.8.1 +:release_date: 2024-04-23 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.8.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v3.9.0.asciidoc b/docs/versioned-plugins/inputs/http-v3.9.0.asciidoc new file mode 100644 index 000000000..417a8ea96 --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.9.0.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.9.0 +:release_date: 2024-08-28 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.9.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v3.9.1.asciidoc b/docs/versioned-plugins/inputs/http-v3.9.1.asciidoc new file mode 100644 index 000000000..0e8809522 --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.9.1.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.9.1 +:release_date: 2024-10-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.9.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v3.9.2.asciidoc b/docs/versioned-plugins/inputs/http-v3.9.2.asciidoc new file mode 100644 index 000000000..211703dbc --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v3.9.2.asciidoc @@ -0,0 +1,540 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.9.2 +:release_date: 2024-11-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v3.9.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |{logstash-ref}/configuration-file-structure.html#path[path]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-tls_max_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tls_min_version>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-verify_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "peer", "force_peer"]`|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + +[id="{version}-plugins-{type}s-{plugin}-cipher_suites"] +===== `cipher_suites` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + +The list of cipher suites to use, listed by priorities. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS keystore to validate the client's certificates + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.7.0, Use <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> instead] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Events are, by default, sent in plain text. You can +enable encryption by setting `ssl` to true and configuring +the `ssl_certificate` and `ssl_key` options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify_mode"] +===== `ssl_verify_mode` +deprecated[3.7.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +By default the server doesn't do any client verification. + +`peer` will make the server ask the client to provide a certificate. +If the client provides a certificate, it will be validated. + +`force_peer` will make the server ask the client to provide a certificate. +If the client doesn't provide a certificate, the connection will be closed. + +This option needs to be used with <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + +[id="{version}-plugins-{type}s-{plugin}-tls_max_version"] +===== `tls_max_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The maximum TLS version allowed for the encrypted connections. +The value must be the one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-tls_min_version"] +===== `tls_min_version` +deprecated[3.6.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +The minimum TLS version allowed for the encrypted connections. +The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLSv1.3 + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-verify_mode"] +===== `verify_mode` +deprecated[3.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verify_mode>>] + + * Value can be any of: `none`, `peer`, `force_peer` + * Default value is `"none"` + +Set the client certificate verification method. Valid methods: none, peer, force_peer + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v4.0.0.asciidoc b/docs/versioned-plugins/inputs/http-v4.0.0.asciidoc new file mode 100644 index 000000000..fe47fb95c --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v4.0.0.asciidoc @@ -0,0 +1,478 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.0 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v4.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Input Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v4.1.0.asciidoc b/docs/versioned-plugins/inputs/http-v4.1.0.asciidoc new file mode 100644 index 000000000..348ae57db --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v4.1.0.asciidoc @@ -0,0 +1,478 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.0 +:release_date: 2024-12-19 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v4.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Input Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v4.1.1.asciidoc b/docs/versioned-plugins/inputs/http-v4.1.1.asciidoc new file mode 100644 index 000000000..161a36616 --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v4.1.1.asciidoc @@ -0,0 +1,478 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.1 +:release_date: 2025-02-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v4.1.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Input Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v4.1.2.asciidoc b/docs/versioned-plugins/inputs/http-v4.1.2.asciidoc new file mode 100644 index 000000000..84fbfbe91 --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v4.1.2.asciidoc @@ -0,0 +1,478 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.2 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v4.1.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Input Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http-v4.1.3.asciidoc b/docs/versioned-plugins/inputs/http-v4.1.3.asciidoc new file mode 100644 index 000000000..404a315f9 --- /dev/null +++ b/docs/versioned-plugins/inputs/http-v4.1.3.asciidoc @@ -0,0 +1,478 @@ +:plugin: http +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.3 +:release_date: 2025-09-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http/blob/v4.1.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Using this input you can receive single or multiline events over http(s). +Applications can send an HTTP request to the endpoint started by this input and +Logstash will convert it into an event for subsequent processing. Users +can pass plain text, JSON, or any formatted data and use a corresponding codec with this +input. For Content-Type `application/json` the `json` codec is used, but for all other +data formats, `plain` codec is used. + +This input can also be used to receive webhook requests to integrate with other services +and applications. By taking advantage of the vast plugin ecosystem available in Logstash +you can trigger actionable events right from your application. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-additional_codecs>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_pending_requests>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-response_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-response_code>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[200, 201, 202, 204]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-additional_codecs"] +===== `additional_codecs` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"application/json"=>"json"}` + +Apply specific codecs for specific content types. +The default codec will be applied only after this list is checked +and no codec for the request's content-type is found + + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +**Sample output: ECS disabled** +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The host or ip to bind + + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `8080` + +The TCP port to bind to + +[id="{version}-plugins-{type}s-{plugin}-max_content_length"] +===== `max_content_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 104857600 + +The max content of an HTTP request in bytes. It defaults to 100mb. + +[id="{version}-plugins-{type}s-{plugin}-max_pending_requests"] +===== `max_pending_requests` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 200 + +Maximum number of incoming requests to store in a temporary queue before being processed by worker threads. +If a request arrives and the queue is full a 429 response will be returned immediately. +This queue exists to deal with micro bursts of events and to improve overall throughput, +so it should be changed very carefully as it can lead to memory pressure and impact performance. +If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the +Persistent Queue for the logstash pipeline. + +[id="{version}-plugins-{type}s-{plugin}-response_code"] +===== `response_code` + + * Value can be any of: 200, 201, 202, 204 + * Default value is `200` + +The HTTP return code if the request is processed successfully. + +Other return codes may happen in the case of an error condition, such as +invalid credentials (401), internal errors (503) or backpressure (429). + +If 204 (No Content) is set, the response body will not be sent in the response. + +[id="{version}-plugins-{type}s-{plugin}-response_headers"] +===== `response_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{"Content-Type"=>"text/plain"}` + +specify a custom set of response headers + +[id="{version}-plugins-{type}s-{plugin}-remote_host_target_field"] +===== `remote_host_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled + +specify a target field for the client host of the http request + +[id="{version}-plugins-{type}s-{plugin}-request_headers_target_field"] +===== `request_headers_target_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled + +specify target field for the client host of the http request + + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificates against these authorities. +You can define multiple files or paths. All the certificates will +be read and added to the trust store. You need to configure the <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +to `optional` or `required` to enable the verification. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `['TLS_AES_256_GCM_SHA384', 'TLS_AES_128_GCM_SHA256', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256']` + +The list of cipher suites to use, listed by priorities. +This default list applies for OpenJDK 11.0.14 and higher. +For older JDK versions, the default list includes only suites supported by that version. +For example, the ChaCha20 family of ciphers is not supported in older versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: `none`, `optional`, `required` +* Default value is `"none"` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. Defaults to `none`, which disables the client authentication. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Events are, by default, sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring +the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> and <<{version}-plugins-{type}s-{plugin}-ssl_key>> options. + +[id="{version}-plugins-{type}s-{plugin}-ssl_handshake_timeout"] +===== `ssl_handshake_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Time in milliseconds for an incomplete ssl handshake to timeout + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[OpenSSL] +for more information. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +SSL key passphrase to use. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore file that contains a private key and certificate. +It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the JKS keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path for the keystore that contains the certificates to trust. It must be either a Java keystore (jks) or a PKCS#12 file. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is number of processors + +Number of threads to use for both accepting connections and handling requests + + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username for basic authorization + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Input Obsolete Configuration Options + +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cipher_suites |<<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +| tls_max_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| tls_min_version |<<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> +| verify_mode |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http_poller-index.asciidoc b/docs/versioned-plugins/inputs/http_poller-index.asciidoc index e0fd4c124..38230ecdd 100644 --- a/docs/versioned-plugins/inputs/http_poller-index.asciidoc +++ b/docs/versioned-plugins/inputs/http_poller-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-12-18 +| <> | 2025-03-27 +| <> | 2024-06-19 | <> | 2023-11-13 | <> | 2023-09-01 | <> | 2022-06-14 @@ -29,6 +32,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-08 |======================================================================= +include::http_poller-v6.0.0.asciidoc[] +include::http_poller-v5.6.1.asciidoc[] +include::http_poller-v5.6.0.asciidoc[] include::http_poller-v5.5.1.asciidoc[] include::http_poller-v5.5.0.asciidoc[] include::http_poller-v5.4.0.asciidoc[] diff --git a/docs/versioned-plugins/inputs/http_poller-v5.6.0.asciidoc b/docs/versioned-plugins/inputs/http_poller-v5.6.0.asciidoc new file mode 100644 index 000000000..5f4c7ef26 --- /dev/null +++ b/docs/versioned-plugins/inputs/http_poller-v5.6.0.asciidoc @@ -0,0 +1,669 @@ +:plugin: http_poller +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.0 +:release_date: 2024-06-19 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http_poller/blob/v5.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http_poller input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and +send them on their merry way. The idea behind this plugins came from a need to read springboot +metrics endpoint, instead of configuring jmx to monitor my java application memory/gc/ etc. + +==== Example +Reads from a list of urls and decodes the body of the response with a codec. +The config should look like this: + +[source,ruby] +---------------------------------- +input { + http_poller { + urls => { + test1 => "http://localhost:9200" + test2 => { + # Supports all options supported by ruby's Manticore HTTP client + method => get + user => "AzureDiamond" + password => "hunter2" + url => "http://localhost:9200/_cluster/health" + headers => { + Accept => "application/json" + } + } + } + request_timeout => 60 + # Supports "cron", "every", "at" and "in" schedules by rufus scheduler + schedule => { cron => "* * * * * UTC"} + codec => "json" + # A hash of request metadata info (timing, response headers, etc.) will be sent here + metadata_target => "http_poller_metadata" + } +} + +output { + stdout { + codec => rubydebug + } +} +---------------------------------- + +Using the HTTP poller with custom a custom CA or self signed cert. + +If you have a self signed cert you will need to convert your server's certificate to a valid# `.jks` or `.p12` file. An easy way to do it is to run the following one-liner, substituting your server's URL for the placeholder `MYURL` and `MYPORT`. + +[source,ruby] +---------------------------------- +openssl s_client -showcerts -connect MYURL:MYPORT /dev/null|openssl x509 -outform PEM > downloaded_cert.pem; keytool -import -alias test -file downloaded_cert.pem -keystore downloaded_truststore.jks +---------------------------------- + +The above snippet will create two files `downloaded_cert.pem` and `downloaded_truststore.jks`. You will be prompted to set a password for the `jks` file during this process. To configure logstash use a config like the one that follows. + + +[source,ruby] +---------------------------------- + http_poller { + urls => { + myurl => "https://myhostname:1234" + } + truststore => "/path/to/downloaded_truststore.jks" + truststore_password => "mypassword" + schedule => { cron => "* * * * * UTC"} + } +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +This input will add metadata about the HTTP connection itself to each event. + +When ECS compatibility is disabled, metadata was added to a variety of non-standard top-level fields, which has the potential to create confusion and schema conflicts downstream. + +With ECS Compatibility Mode, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-metadata_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|Yes +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-urls>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|Yes +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We highly recommend NOT setting this value +to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry! +Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="{version}-plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="{version}-plugins-{type}s-{plugin}-client_key"] +===== `client_key` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_key>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: unstructured data added at root level +** `v1`: uses `error`, `url` and `http` fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +Example output: + +**Sample output: ECS disabled** +[source,text] +----- +{ + "http_poller_data" => { + "@version" => "1", + "@timestamp" => 2021-01-01T00:43:22.388Z, + "status" => "UP" + }, + "@version" => "1", + "@timestamp" => 2021-01-01T00:43:22.389Z, +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "http_poller_data" => { + "status" => "UP", + "@version" => "1", + "event" => { + "original" => "{\"status\":\"UP\"}" + }, + "@timestamp" => 2021-01-01T00:40:59.558Z + }, + "@version" => "1", + "@timestamp" => 2021-01-01T00:40:59.559Z +} +----- + +**Sample error output: ECS enabled** +[source,text] +---- +{ + "@timestamp" => 2021-07-09T09:53:48.721Z, + "@version" => "1", + "host" => { + "hostname" => "MacBook-Pro" + }, + "http" => { + "request" => { + "method" => "get" + } + }, + "event" => { + "duration" => 259019 + }, + "error" => { + "stack_trace" => nil, + "message" => "Connection refused (Connection refused)" + }, + "url" => { + "full" => "http://localhost:8080/actuator/health" + }, + "tags" => [ + [0] "_http_request_failure" + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-metadata_target"] +===== `metadata_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@metadata"` + +If you'd like to work with the request/response metadata. +Set this value to the name of the field you'd like to store a nested +hash of metadata. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to be used in conjunction with <<{version}-plugins-{type}s-{plugin}-user>> for HTTP authentication. + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Timeout (in seconds) for the entire request. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Schedule of when to periodically poll from the urls +Format: A hash with + + key: "cron" | "every" | "in" | "at" + + value: string +Examples: + a) { "every" => "1h" } + b) { "cron" => "* * * * * UTC" } +See: rufus/scheduler for details about different schedule options and value string format + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event. + +TIP: When ECS is enabled, set `target` in the codec (if the codec has a `target` option). +Example: `codec => json { target => "TARGET_FIELD_NAME" }` + + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-urls"] +===== `urls` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +A Hash of urls in this format : `"name" => "url"`. +The name and the url will be passed in the outputted event. + +The values in urls can be either: + +* a string url (which will be issued as an HTTP GET). +* a sub-hash containing many useful keys provided by the Manticore backend: +** url: the String url +** method: (optional) the HTTP method to use (defaults to GET) +** user: (optional) the HTTP Basic Auth user. The user must be under + an auth sub-hash for Manticore, but this plugin also accepts it either way. +** password: (optional) the HTTP Basic Auth password. The password + must be under an auth sub-hash for Manticore, but this plugin accepts it either way. +** headers: a hash containing key-value pairs of headers. +** body: a string (supported only on POST and PUT requests) +** possibly other options mentioned in the + https://www.rubydoc.info/github/cheald/manticore/Manticore/Client#http-instance_method[Manticore docs]. + Note that Manticore options that are not explicitly documented above are not + thoroughly tested and therefore liable to break in unexpected ways if we + replace the backend. + +*Notes:* + +* Passwords specified as a part of `urls` are prone to exposure in plugin log output. +The plugin does not declare them as passwords, and therefore doesn't wrap them in +leak-reducing wrappers as we do elsewhere. +* We don't guarantee that boolean-type options like Manticore's `follow_redirects` are supported +correctly. The strings `true` or `false` may get passed through, and in ruby any +string is "truthy." +* Our implementation of this plugin precludes the ability to specify auth[:eager] +as anything other than true + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL. +If you set this you must also set the <<{version}-plugins-{type}s-{plugin}-password>> option. + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons' HTTP implementation. Here's how the +https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http_poller-v5.6.1.asciidoc b/docs/versioned-plugins/inputs/http_poller-v5.6.1.asciidoc new file mode 100644 index 000000000..1582d86af --- /dev/null +++ b/docs/versioned-plugins/inputs/http_poller-v5.6.1.asciidoc @@ -0,0 +1,669 @@ +:plugin: http_poller +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.1 +:release_date: 2025-03-27 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http_poller/blob/v5.6.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http_poller input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and +send them on their merry way. The idea behind this plugins came from a need to read springboot +metrics endpoint, instead of configuring jmx to monitor my java application memory/gc/ etc. + +==== Example +Reads from a list of urls and decodes the body of the response with a codec. +The config should look like this: + +[source,ruby] +---------------------------------- +input { + http_poller { + urls => { + test1 => "http://localhost:9200" + test2 => { + # Supports all options supported by ruby's Manticore HTTP client + method => get + user => "AzureDiamond" + password => "hunter2" + url => "http://localhost:9200/_cluster/health" + headers => { + Accept => "application/json" + } + } + } + request_timeout => 60 + # Supports "cron", "every", "at" and "in" schedules by rufus scheduler + schedule => { cron => "* * * * * UTC"} + codec => "json" + # A hash of request metadata info (timing, response headers, etc.) will be sent here + metadata_target => "http_poller_metadata" + } +} + +output { + stdout { + codec => rubydebug + } +} +---------------------------------- + +Using the HTTP poller with custom a custom CA or self signed cert. + +If you have a self signed cert you will need to convert your server's certificate to a valid# `.jks` or `.p12` file. An easy way to do it is to run the following one-liner, substituting your server's URL for the placeholder `MYURL` and `MYPORT`. + +[source,ruby] +---------------------------------- +openssl s_client -showcerts -connect MYURL:MYPORT /dev/null|openssl x509 -outform PEM > downloaded_cert.pem; keytool -import -alias test -file downloaded_cert.pem -keystore downloaded_truststore.jks +---------------------------------- + +The above snippet will create two files `downloaded_cert.pem` and `downloaded_truststore.jks`. You will be prompted to set a password for the `jks` file during this process. To configure logstash use a config like the one that follows. + + +[source,ruby] +---------------------------------- + http_poller { + urls => { + myurl => "https://myhostname:1234" + } + truststore => "/path/to/downloaded_truststore.jks" + truststore_password => "mypassword" + schedule => { cron => "* * * * * UTC"} + } +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +This input will add metadata about the HTTP connection itself to each event. + +When ECS compatibility is disabled, metadata was added to a variety of non-standard top-level fields, which has the potential to create confusion and schema conflicts downstream. + +With ECS Compatibility Mode, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-metadata_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|Yes +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-urls>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|Yes +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We highly recommend NOT setting this value +to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry! +Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="{version}-plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="{version}-plugins-{type}s-{plugin}-client_key"] +===== `client_key` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_key>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: unstructured data added at root level +** `v1`: uses `error`, `url` and `http` fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +Example output: + +**Sample output: ECS disabled** +[source,text] +----- +{ + "http_poller_data" => { + "@version" => "1", + "@timestamp" => 2021-01-01T00:43:22.388Z, + "status" => "UP" + }, + "@version" => "1", + "@timestamp" => 2021-01-01T00:43:22.389Z, +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "http_poller_data" => { + "status" => "UP", + "@version" => "1", + "event" => { + "original" => "{\"status\":\"UP\"}" + }, + "@timestamp" => 2021-01-01T00:40:59.558Z + }, + "@version" => "1", + "@timestamp" => 2021-01-01T00:40:59.559Z +} +----- + +**Sample error output: ECS enabled** +[source,text] +---- +{ + "@timestamp" => 2021-07-09T09:53:48.721Z, + "@version" => "1", + "host" => { + "hostname" => "MacBook-Pro" + }, + "http" => { + "request" => { + "method" => "get" + } + }, + "event" => { + "duration" => 259019 + }, + "error" => { + "stack_trace" => nil, + "message" => "Connection refused (Connection refused)" + }, + "url" => { + "full" => "http://localhost:8080/actuator/health" + }, + "tags" => [ + [0] "_http_request_failure" + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-metadata_target"] +===== `metadata_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@metadata"` + +If you'd like to work with the request/response metadata. +Set this value to the name of the field you'd like to store a nested +hash of metadata. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to be used in conjunction with <<{version}-plugins-{type}s-{plugin}-user>> for HTTP authentication. + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Timeout (in seconds) for the entire request. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Schedule of when to periodically poll from the urls +Format: A hash with + + key: "cron" | "every" | "in" | "at" + + value: string +Examples: + a) { "every" => "1h" } + b) { "cron" => "* * * * * UTC" } +See: rufus/scheduler for details about different schedule options and value string format + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event. + +TIP: When ECS is enabled, set `target` in the codec (if the codec has a `target` option). +Example: `codec => json { target => "TARGET_FIELD_NAME" }` + + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` +deprecated[5.5.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-urls"] +===== `urls` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +A Hash of urls in this format : `"name" => "url"`. +The name and the url will be passed in the outputted event. + +The values in urls can be either: + +* a string url (which will be issued as an HTTP GET). +* a sub-hash containing many useful keys provided by the Manticore backend: +** url: the String url +** method: (optional) the HTTP method to use (defaults to GET) +** user: (optional) the HTTP Basic Auth user. The user must be under + an auth sub-hash for Manticore, but this plugin also accepts it either way. +** password: (optional) the HTTP Basic Auth password. The password + must be under an auth sub-hash for Manticore, but this plugin accepts it either way. +** headers: a hash containing key-value pairs of headers. +** body: a string (supported only on POST and PUT requests) +** possibly other options mentioned in the + https://www.rubydoc.info/github/cheald/manticore/Manticore/Client#http-instance_method[Manticore docs]. + Note that Manticore options that are not explicitly documented above are not + thoroughly tested and therefore liable to break in unexpected ways if we + replace the backend. + +*Notes:* + +* Passwords specified as a part of `urls` are prone to exposure in plugin log output. +The plugin does not declare them as passwords, and therefore doesn't wrap them in +leak-reducing wrappers as we do elsewhere. +* We don't guarantee that boolean-type options like Manticore's `follow_redirects` are supported +correctly. The strings `true` or `false` may get passed through, and in ruby any +string is "truthy." +* Our implementation of this plugin precludes the ability to specify auth[:eager] +as anything other than true + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL. +If you set this you must also set the <<{version}-plugins-{type}s-{plugin}-password>> option. + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons' HTTP implementation. Here's how the +https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/http_poller-v6.0.0.asciidoc b/docs/versioned-plugins/inputs/http_poller-v6.0.0.asciidoc new file mode 100644 index 000000000..6d5ae3621 --- /dev/null +++ b/docs/versioned-plugins/inputs/http_poller-v6.0.0.asciidoc @@ -0,0 +1,602 @@ +:plugin: http_poller +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.0.0 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-http_poller/blob/v6.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http_poller input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and +send them on their merry way. The idea behind this plugins came from a need to read springboot +metrics endpoint, instead of configuring jmx to monitor my java application memory/gc/ etc. + +==== Example +Reads from a list of urls and decodes the body of the response with a codec. +The config should look like this: + +[source,ruby] +---------------------------------- +input { + http_poller { + urls => { + test1 => "http://localhost:9200" + test2 => { + # Supports all options supported by ruby's Manticore HTTP client + method => get + user => "AzureDiamond" + password => "hunter2" + url => "http://localhost:9200/_cluster/health" + headers => { + Accept => "application/json" + } + } + } + request_timeout => 60 + # Supports "cron", "every", "at" and "in" schedules by rufus scheduler + schedule => { cron => "* * * * * UTC"} + codec => "json" + # A hash of request metadata info (timing, response headers, etc.) will be sent here + metadata_target => "http_poller_metadata" + } +} + +output { + stdout { + codec => rubydebug + } +} +---------------------------------- + +Using the HTTP poller with custom a custom CA or self signed cert. + +If you have a self signed cert you will need to convert your server's certificate to a valid# `.jks` or `.p12` file. An easy way to do it is to run the following one-liner, substituting your server's URL for the placeholder `MYURL` and `MYPORT`. + +[source,ruby] +---------------------------------- +openssl s_client -showcerts -connect MYURL:MYPORT /dev/null|openssl x509 -outform PEM > downloaded_cert.pem; keytool -import -alias test -file downloaded_cert.pem -keystore downloaded_truststore.jks +---------------------------------- + +The above snippet will create two files `downloaded_cert.pem` and `downloaded_truststore.jks`. You will be prompted to set a password for the `jks` file during this process. To configure logstash use a config like the one that follows. + + +[source,ruby] +---------------------------------- + http_poller { + urls => { + myurl => "https://myhostname:1234" + } + truststore => "/path/to/downloaded_truststore.jks" + truststore_password => "mypassword" + schedule => { cron => "* * * * * UTC"} + } +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +This input will add metadata about the HTTP connection itself to each event. + +When ECS compatibility is disabled, metadata was added to a variety of non-standard top-level fields, which has the potential to create confusion and schema conflicts downstream. + +With ECS Compatibility Mode, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +NOTE: As of version `6.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|Yes +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-urls>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|Yes +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We highly recommend NOT setting this value +to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry! +Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: unstructured data added at root level +** `v1`: uses `error`, `url` and `http` fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +Example output: + +**Sample output: ECS disabled** +[source,text] +----- +{ + "http_poller_data" => { + "@version" => "1", + "@timestamp" => 2021-01-01T00:43:22.388Z, + "status" => "UP" + }, + "@version" => "1", + "@timestamp" => 2021-01-01T00:43:22.389Z, +} +----- + +**Sample output: ECS enabled** +[source,text] +----- +{ + "http_poller_data" => { + "status" => "UP", + "@version" => "1", + "event" => { + "original" => "{\"status\":\"UP\"}" + }, + "@timestamp" => 2021-01-01T00:40:59.558Z + }, + "@version" => "1", + "@timestamp" => 2021-01-01T00:40:59.559Z +} +----- + +**Sample error output: ECS enabled** +[source,text] +---- +{ + "@timestamp" => 2021-07-09T09:53:48.721Z, + "@version" => "1", + "host" => { + "hostname" => "MacBook-Pro" + }, + "http" => { + "request" => { + "method" => "get" + } + }, + "event" => { + "duration" => 259019 + }, + "error" => { + "stack_trace" => nil, + "message" => "Connection refused (Connection refused)" + }, + "url" => { + "full" => "http://localhost:8080/actuator/health" + }, + "tags" => [ + [0] "_http_request_failure" + ] +} +---- + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-metadata_target"] +===== `metadata_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@metadata"` + +If you'd like to work with the request/response metadata. +Set this value to the name of the field you'd like to store a nested +hash of metadata. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to be used in conjunction with <<{version}-plugins-{type}s-{plugin}-user>> for HTTP authentication. + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Timeout (in seconds) for the entire request. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried. + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Schedule of when to periodically poll from the urls +Format: A hash with + + key: "cron" | "every" | "in" | "at" + + value: string +Examples: + a) { "every" => "1h" } + b) { "cron" => "* * * * * UTC" } +See: rufus/scheduler for details about different schedule options and value string format + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event. + +TIP: When ECS is enabled, set `target` in the codec (if the codec has a `target` option). +Example: `codec => json { target => "TARGET_FIELD_NAME" }` + +[id="{version}-plugins-{type}s-{plugin}-urls"] +===== `urls` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +A Hash of urls in this format : `"name" => "url"`. +The name and the url will be passed in the outputted event. + +The values in urls can be either: + +* a string url (which will be issued as an HTTP GET). +* a sub-hash containing many useful keys provided by the Manticore backend: +** url: the String url +** method: (optional) the HTTP method to use (defaults to GET) +** user: (optional) the HTTP Basic Auth user. The user must be under + an auth sub-hash for Manticore, but this plugin also accepts it either way. +** password: (optional) the HTTP Basic Auth password. The password + must be under an auth sub-hash for Manticore, but this plugin accepts it either way. +** headers: a hash containing key-value pairs of headers. +** body: a string (supported only on POST and PUT requests) +** possibly other options mentioned in the + https://www.rubydoc.info/github/cheald/manticore/Manticore/Client#http-instance_method[Manticore docs]. + Note that Manticore options that are not explicitly documented above are not + thoroughly tested and therefore liable to break in unexpected ways if we + replace the backend. + +*Notes:* + +* Passwords specified as a part of `urls` are prone to exposure in plugin log output. +The plugin does not declare them as passwords, and therefore doesn't wrap them in +leak-reducing wrappers as we do elsewhere. +* We don't guarantee that boolean-type options like Manticore's `follow_redirects` are supported +correctly. The strings `true` or `false` may get passed through, and in ruby any +string is "truthy." +* Our implementation of this plugin precludes the ability to specify auth[:eager] +as anything other than true + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL. +If you set this you must also set the <<{version}-plugins-{type}s-{plugin}-password>> option. + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons' HTTP implementation. Here's how the +https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Poller Input Obsolete Configuration Options + +WARNING: As of version `6.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| client_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| client_key |<<{version}-plugins-{type}s-{plugin}-ssl_key>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| keystore_type |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| truststore |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +| truststore_type |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-index.asciidoc b/docs/versioned-plugins/inputs/jdbc-index.asciidoc index 9c6cb1e8b..3d968e23d 100644 --- a/docs/versioned-plugins/inputs/jdbc-index.asciidoc +++ b/docs/versioned-plugins/inputs/jdbc-index.asciidoc @@ -5,6 +5,16 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-05-30 +| <> | 2025-03-07 +| <> | 2024-12-23 +| <> | 2024-12-03 +| <> | 2024-08-09 +| <> | 2024-05-23 +| <> | 2024-05-08 +| <> | 2024-03-21 +| <> | 2024-02-13 +| <> | 2024-01-31 | <> | 2023-10-03 | <> | 2023-09-04 | <> | 2023-06-15 @@ -62,6 +72,16 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::jdbc-v5.6.0.asciidoc[] +include::jdbc-v5.5.3.asciidoc[] +include::jdbc-v5.5.2.asciidoc[] +include::jdbc-v5.5.1.asciidoc[] +include::jdbc-v5.5.0.asciidoc[] +include::jdbc-v5.4.11.asciidoc[] +include::jdbc-v5.4.10.asciidoc[] +include::jdbc-v5.4.9.asciidoc[] +include::jdbc-v5.4.8.asciidoc[] +include::jdbc-v5.4.7.asciidoc[] include::jdbc-v5.4.6.asciidoc[] include::jdbc-v5.4.5.asciidoc[] include::jdbc-v5.4.4.asciidoc[] diff --git a/docs/versioned-plugins/inputs/jdbc-v5.4.10.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.4.10.asciidoc new file mode 100644 index 000000000..abe3bf30a --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.4.10.asciidoc @@ -0,0 +1,674 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.10 +:release_date: 2024-05-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.10/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.4.11.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.4.11.asciidoc new file mode 100644 index 000000000..0049e2c70 --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.4.11.asciidoc @@ -0,0 +1,674 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.11 +:release_date: 2024-05-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.11/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.4.7.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.4.7.asciidoc new file mode 100644 index 000000000..286f1aa98 --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.4.7.asciidoc @@ -0,0 +1,651 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.7 +:release_date: 2024-01-31 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.4.8.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.4.8.asciidoc new file mode 100644 index 000000000..a5b09456d --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.4.8.asciidoc @@ -0,0 +1,651 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.8 +:release_date: 2024-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.4.9.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.4.9.asciidoc new file mode 100644 index 000000000..ca09cc6ed --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.4.9.asciidoc @@ -0,0 +1,651 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.9 +:release_date: 2024-03-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.5.0.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.5.0.asciidoc new file mode 100644 index 000000000..32883a538 --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.5.0.asciidoc @@ -0,0 +1,674 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.0 +:release_date: 2024-08-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.5.1.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.5.1.asciidoc new file mode 100644 index 000000000..7e22cd5e8 --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.5.1.asciidoc @@ -0,0 +1,692 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.1 +:release_date: 2024-12-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts"] +===== `statement_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try executing a statement. + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time"] +===== `statement_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between statement execution attempts. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.5.2.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.5.2.asciidoc new file mode 100644 index 000000000..b104a1217 --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.5.2.asciidoc @@ -0,0 +1,692 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.2 +:release_date: 2024-12-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts"] +===== `statement_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try executing a statement. + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time"] +===== `statement_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between statement execution attempts. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.5.3.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.5.3.asciidoc new file mode 100644 index 000000000..f44627b82 --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.5.3.asciidoc @@ -0,0 +1,692 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.3 +:release_date: 2025-03-07 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). + +Examples: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format +for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no schedule is given, then the statement is run +exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts"] +===== `statement_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try executing a statement. + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time"] +===== `statement_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between statement execution attempts. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jdbc-v5.6.0.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.6.0.asciidoc new file mode 100644 index 000000000..6375449b6 --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.6.0.asciidoc @@ -0,0 +1,724 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.0 +:release_date: 2025-05-30 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is either cron-like with some extensions specific to Rufus (e.g. timezone support ) if using the `schedule` option or periodic when using `period` or `interval` option. + +Examples for `schedule`: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Examples for `period` or `interval`: + +|========================================================== +| `1m` | will execute every minute +| `3h10m` | will execute every three hours and 10 minutes +|========================================================== + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +`interval` jobs trigger, execute and then trigger again after the interval elapsed. + +`period` jobs try to trigger following the frequency they were scheduled with. + +You can only use one of `interval`, `period` or `schedule` at the same time. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-period>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This takes a string in the form of `1h`, `1m`, to denote a time interval. `interval` jobs trigger, execute and trigger again after the provided time interval has elapsed. + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-period"] +===== `period` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This takes a string in the form of `1h`, `1m`, to denote a time interval. `period` jobs try hard to trigger following the frequency they were scheduled with. + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts"] +===== `statement_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try executing a statement. + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time"] +===== `statement_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between statement execution attempts. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jms-index.asciidoc b/docs/versioned-plugins/inputs/jms-index.asciidoc index fcd3b395c..abd0b4886 100644 --- a/docs/versioned-plugins/inputs/jms-index.asciidoc +++ b/docs/versioned-plugins/inputs/jms-index.asciidoc @@ -5,6 +5,8 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-09-08 +| <> | 2025-03-07 | <> | 2022-06-13 | <> | 2022-01-25 | <> | 2021-11-19 @@ -18,6 +20,8 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::jms-v3.3.1.asciidoc[] +include::jms-v3.3.0.asciidoc[] include::jms-v3.2.2.asciidoc[] include::jms-v3.2.1.asciidoc[] include::jms-v3.2.0.asciidoc[] diff --git a/docs/versioned-plugins/inputs/jms-v3.3.0.asciidoc b/docs/versioned-plugins/inputs/jms-v3.3.0.asciidoc new file mode 100644 index 000000000..4710ff3fe --- /dev/null +++ b/docs/versioned-plugins/inputs/jms-v3.3.0.asciidoc @@ -0,0 +1,811 @@ +:plugin: jms +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.3.0 +:release_date: 2025-03-07 +:changelog_url: https://github.com/logstash-plugins/logstash-input-jms/blob/v3.3.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jms input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events from a Jms Broker. Supports both Jms Queues and Topics. + +For more information about Jms, see . +For more information about the Ruby Gem used, see . + + +JMS configurations can be done either entirely in the Logstash configuration file, or in a mixture of the Logstash + configuration file, and a specified yaml file. Simple configurations that do not need to make calls to implementation + specific methods on the connection factory can be specified entirely in the Logstash configuration, whereas more complex + configurations, should also use the combination of yaml file and Logstash configuration. + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +JMS data is application specific. ECS compliance for fields depends on the use case. +The plugin includes sensible defaults that change based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>. +When ECS compatibility is disabled, headers, properties, and payload are targeted at the root level of the event to maintain compatibility with legacy usage of this plugin. +When targeting an ECS version, headers and properties target `@metadata` sub-fields unless configured otherwise in order to avoid conflict with ECS fields. +See <<{version}-plugins-{type}s-{plugin}-headers_target>>, <<{version}-plugins-{type}s-{plugin}-properties_target>> and <<{version}-plugins-{type}s-{plugin}-target>>. + +==== Sample Configuration using Logstash Configuration Only + +Configurations can be configured either entirely in Logstash configuration, or via a combination of Logstash configuration +and yaml file, which can be useful for sharing similar configurations across multiple inputs and outputs. +The JMS plugin can also be configured using JNDI if desired. + +===== Logstash Confuguration for Non-JNDI Connection +[source,ruby] +-------------------------------------------------- + input + { + jms { + broker_url => 'failover:(tcp://host1:61616,tcp://host2:61616)?initialReconnectDelay=100' <1> + destination => 'myqueue' <2> + factory => 'org.apache.activemq.ActiveMQConnectionFactory' <3> + pub_sub => false <4> + use_jms_timestamp => false <5> + # JMS provider credentials if needed <6> + username => 'username' + password => 'secret' + # JMS provider keystore and truststore details <7> + keystore => '/Users/logstash-user/security/keystore.jks' + keystore_password => 'another_secret' + truststore => '/Users/logstash-user/security/truststore.jks' + truststore_password => 'yet_another_secret' + # Parts of the JMS message to be included <8> + include_headers => false + include_properties => false + include_body => true + # Message selector + selector => "string_property = 'this' OR int_property < 3" <9> + # Connection factory specific settings + factory_settings => { <10> + exclusive_consumer => true + } + # Jar Files to include + require_jars => ['/usr/share/jms/activemq-all-5.15.9.jar'] <11> + } + } +-------------------------------------------------- + +<1> Url of the broker to connect to. Please consult your JMS provider documentation for the exact syntax to use + here, including how to enable failover. +<2> Name of the topic or queue that the plugin will listen to events from. +<3> Full name (including package name) of Java connection factory used to create a connection with your JMS provider. +<4> Determines whether the event source is a queue or a topic, set to `true` for topic, `false` for queue. +<5> Determines whether the JMSTimestamp header is used to populate the `@timestamp` field. +<6> Credentials to use when connecting to the JMS provider, if required. +<7> Keystore and Truststore to use when connecting to the JMS provider, if required. +<8> Parts of the JMS Message to include in the event - headers, properties and the message body can be included or + excluded from the event. +<9> Message selector: Use this to filter messages to be processed. The whole selector query should be double-quoted, + string property values should be single quoted, and numeric property vaues should not be quoted. + See JMS provider documentation for exact syntax. +<10> Additional settings that are set directly on the ConnectionFactory object can be added here. +<11> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + +===== Logstash Configuration for JNDI Connection +[source,ruby] +-------------------------------------------------- + input { + jms { + # Logstash Configuration Settings. <1> + include_headers => false + include_properties => false + include_body => true + use_jms_timestamp => false + destination => "myqueue" + pub_sub => false + # JNDI Settings + jndi_name => /jms/cf/default <2> + jndi_context => { <3> + 'java.naming.factory.initial' => com.solacesystems.jndi.SolJNDIInitialContextFactory + 'java.naming.security.principal' => solace-cloud-client@user + 'java.naming.provider.url' => tcp://address.messaging.solace.cloud:20608 + 'java.naming.security.credentials' => verysecret + } + # Jar files to be imported + require_jars=> ['/usr/share/jms/commons-lang-2.6.jar', <4> + '/usr/share/jms/sol-jms-10.5.0.jar', + '/usr/share/jms/geronimo-jms_1.1_spec-1.1.1.jar', + '/usr/share/jms/commons-lang-2.6.jar]' + } + } +-------------------------------------------------- + +<1> Configuration settings. Note that there is no `broker_url` or `username` or `password` defined here - these are + defined in the `jndi_context` hash. +<2> JNDI name for this connection. +<3> JNDI context settings hash. Contains details of how to connect to JNDI server. See your JMS provider documentation for + implementation specific details. +<4> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + + +==== Sample Configuration using Logstash Configuration and Yaml File + +===== Non-JNDI Connection + +This section contains sample configurations for connecting to a JMS provider that is not using JNDI using a combination + of the Logstash configuration and the yaml file + +===== Logstash Configuration for Non-JNDI Connection (for configs including yaml) + +[source,ruby] +-------------------------------------------------- + input { + jms { + # Logstash Configuration File Settings <1> + include_headers => false + include_properties => false + include_body => true + use_jms_timestamp => false + destination => "myqueue" + pub_sub => false + # JMS Provider credentials <2> + username => xxxx + password => xxxx + # Location of yaml file, and which section to use for configuration + yaml_file => "~/jms.yml" <3> + yaml_section => "mybroker" <4> + } + } +-------------------------------------------------- + + +<1> Configuration settings +<2> Username and password for the connection. +<3> Full path to a yaml file containing the definition for the ConnectionFactory. +<4> Section name in the yaml file of the ConnectionFactory for this plugin definition + + +===== Yaml File for Non-JNDI Connection + +[source,yaml] +-------------------------------------------------- +mybroker: <1> + :broker_url: 'ssl://localhost:61617' <2> + :factory: org.apache.activemq.ActiveMQConnectionFactory <3> + :exclusive_consumer: true <4> + :require_jars: <5> + - /usr/share/jms/activemq-all-5.15.9.jar + - /usr/share/jms/log4j-1.2.17.jar +-------------------------------------------------- + +<1> Section name for this broker definition. This should be the value of `yaml_section` in the logstash configuration file. + Note that multiple sections can co-exist in the same yaml file. +<2> Full url of the broker. See your JMS Provider documentation for details. +<3> Full name (including package name) of Java connection factory used to create a connection with your JMS provider. +<4> Implementation specific configuration parameters to be used with the connection factory specified. + in <3>. Each JMS Provider will have its own set of parameters that can be used here. These parameters are mapped to + `set` methods on the provided connection factory, and can be supplied in either 'snake' or 'camel' case. In <4> above, + the `exclusive_consumer` property will call the `setExclusiveConsumer` on the supplied connection factory. See your JMS provider + documentation for implementation specific details. +<5> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + + +===== JNDI Connection + +This section contains sample configurations for connecting to a JMS provider that is using JNDI using a combination + of the Logstash configuration and the yaml file + +===== Logstash Configuration for JNDI Connection (for configs including yaml) + +[source,ruby] +-------------------------------------------------- + input { + jms { + # Logstash specific configuration settings <1> + include_headers => false + include_properties => false + include_body => true + use_jms_timestamp => false + destination => "myqueue" + pub_sub => false + # Location of yaml file, and which section to use for configuration + yaml_file => "~/jms.yml" <2> + yaml_section => "mybroker" <3> + } + } +-------------------------------------------------- + +<1> Configuration settings +<2> Full path to a yaml file containing the definition for the ConnectionFactory. +<3> Section name in the yaml file of the ConnectionFactory for this plugin definition + + +===== Yaml File for JNDI Connection + +[source,yaml] +-------------------------------------------------- + +solace: <1> + :jndi_name: /jms/cf/default <2> + :jndi_context: <3> + java.naming.factory.initial: com.solacesystems.jndi.SolJNDIInitialContextFactory + java.naming.security.principal: solace-cloud-client@user + java.naming.provider.url: tcp://address.messaging.solace.cloud:20608 + java.naming.security.credentials: verysecret + :require_jars: <4> + - /usr/share/jms/commons-lang-2.6.jar + - /usr/share/jms/sol-jms-10.5.0.jar + - /usr/share/jms/geronimo-jms_1.1_spec-1.1.1.jar + - /usr/share/jms/commons-lang-2.6.jar +-------------------------------------------------- + +<1> Section name for this broker definition. This should be the value of `yaml_section` in the Logstash configuration file. +<2> Name of JNDI entry at which the Factory can be found +<3> JNDI context settings. Contains details of how to connect to JNDI server. See your JMS provider documentation for + implementation specific details. +<4> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + + +==== Jar files + +In order to communicate with a JMS broker, the plugin must load the jar files necessary for each client type. +This can be set in the yaml file, or in the main configuration if a yaml file is not necessary. The `require_jars` + setting should include the full path for each jar file required for the client. Eg + +===== Logstash configuration + +[source,ruby] +-------------------------------------------------- + + input { + jms { + : + [snip] + require_jars => ['/usr/share/jms/commons-lang-2.6.jar', + '/usr/share/jms/sol-jms-10.5.0.jar', + '/usr/share/jms/geronimo-jms_1.1_spec-1.1.1.jar', + '/usr/share/jms/commons-lang-2.6.jar'] + } + } +-------------------------------------------------- + +==== Troubleshooting + +This section includes some common issues that a user may have when integrating this plugin with their JMS provider. + +===== Missing Jar files + +The most common issue is missing jar files, which may be jar files provided by the JMS vendor, or jar files that the JMS + vendor requires to run. This issue can manifest in different ways, depending on where the missing jar file is discovered. + +Example log output: + +[source,txt] +----- + Failed to load JMS Connection, likely because a JMS Provider is not on the Logstash classpath or correctly + specified by the plugin's `require_jars` directive + {:exception=>"cannot load Java class javax.jms.DeliveryMode" +----- + +[source,txt] +----- + JMS Consumer Died {:exception=>"Java::JavaxNaming::NoInitialContextException", + :exception_message=>"Cannot instantiate class:" +----- + +[source,txt] +----- + warning: thread "[main]"Java::JavaxJms::JMSException", :exception_message=>"io/netty/channel/epoll/Epoll", + :root_cause=>{:exception=>"Java::JavaLang::NoClassDefFoundError", :exception_message=>"io/netty/channel/epoll/Epoll"} +----- + +If any of these issues occur, check the list of `require_jars` in either the Logstash configuration or yaml configuration + files. + +===== Setting System Properties + +Many JMS providers allow or expect System properties to be set to configure certain properties when using JMS, for +example, the Apache qpid JMS client allows the connection factory lookup to be stored there, and the Solace JMS client +allows many properties, such as number of connection retries to be set as System properties. Any system properties that +are set should be set in the Logstash `jvm.options` file. + +===== Multiple JMS inputs/outputs in the same Logstash process + +The use of multiple JMS consumers and producers in the same Logstash process is unsupported if: + + * System properties need to be different for any of the consumers/producers + * Different keystores or truststores are required for any of the consumers/producers + +===== Message Selectors unexpectedly filtering out all messages + +Incorrect message selector syntax can have two effects - either the syntax is incorrect and the selector parser from the +JMS provider will throw an exception causing the plugin to fail OR the syntax will be accepted, but the messages will +be silently dropped - this can happen with incorrect quoting of string properties in the selector definition. All +selector definitions must be double quoted in the Logstash configuration file, and string property values must be +single quoted, and numeric property values not quoted at all. + + +===== Failed to create Event with MissingConverterException + +Messages from certain JMS providers may contain headers or properties that Logstash cannot interpret, which can lead to +error messages such as: + + +[source,txt] +----- +[2019-11-25T08:04:28,769][ERROR][logstash.inputs.jms ] Failed to create event {:message=>Java::ComSolacesystemsJmsMessage::SolTextMessage: ... +Attributes: {:jms_correlation_id=>"xxxx", :jms_delivery_mode_sym=>:non_persistent, :jms_destination=>"destination", :jms_expiration=>0, :jms_message_id=>"xxxxxx", :jms_priority=>0, :jms_redelivered=>false, :jms_reply_to=>#, :jms_timestamp=>1574669008862, :jms_type=>nil} +Properties: nil, :exception=>org.logstash.MissingConverterException: Missing Converter handling for full class name=com.solacesystems.jms.impl.SolTopicImpl, simple name=SolTopicImpl, :backtrace=>["org.logstash.Valuefier.fallbackConvert(Valuefier.java:98)..."]} +:exception=>org.logstash.MissingConverterException: Missing Converter handling for full class name=com.solacesystems.jms.impl.SolTopicImpl +----- + +To get around this, use the `skip_headers` or `skip_properties` configuration setting to avoid attempting to process +the offending header or property in the message. + +In the example shown above, this attribute is causing the +`MissingConverterException`: + +`jms_reply_to=>#` + + +To avoid this error, the configuration should include the following line: + +[source,ruby] +----- +skip_headers => ["jms_reply_to"] +----- + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jms Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-broker_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-destination>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-durable_subscriber>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-durable_subscriber_client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-durable_subscriber_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-factory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-factory_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-headers_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_body>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_header>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_headers>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_properties>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jndi_context>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jndi_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-oracle_aq_buffered_messages>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-properties_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pub_sub>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-require_jars>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-runner>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-selector>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-skip_headers>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-skip_properties>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-system_properties>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-use_jms_timestamp>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-yaml_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-yaml_section>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-broker_url"] +===== `broker_url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Url to use when connecting to the JMS provider. This is only relevant for non-JNDI configurations. + +[id="{version}-plugins-{type}s-{plugin}-destination"] +===== `destination` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the destination queue or topic to use. + +TIP: If the destination setting doesn't appear to be working properly, try this format: `!`. + +[id="{version}-plugins-{type}s-{plugin}-durable_subscriber"] +===== `durable_subscriber` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * This is `false` by default + * Requires `pub_sub` to be true + +Setting this value to `true` will make subscriptions to topics "durable", which allowing messages that arrived on the +specified topic while Logstash is not running to still be read. Without setting this value, any messages sent to a topic +while Logstash is not actively listening will be lost. A durable subscriber specifies a unique identity consisting of the + topic (`destination`), the client id (`durable_subscriber_client_id`) and subscriber name + (`durable_subscriber_subscriber_name`). See your JMS Provider documentation for any further requirements/limitations + around these settings. + +* Note that a durable subscription can only have one active subscriber at a time. +* Note that this setting is only permitted when `pub_sub` is set to true, and will generate a configuration error otherwise + +[id="{version}-plugins-{type}s-{plugin}-durable_subscriber_client_id"] +===== `durable_subscriber_client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * If `durable_subscriber` is set, the default value for this setting is 'Logstash', otherwise this setting has no + effect + +This represents the value of the client ID for a durable subscribtion, and is only used if `durable_subscriber` is set to `true`. + +[id="{version}-plugins-{type}s-{plugin}-durable_subscriber_name"] +===== `durable_subscriber_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * If `durable_subscriber` is set, the default value for this setting will be the same value as the `destination` + setting, otherwise this setting has no effect. + +This represents the value of the subscriber name for a durable subscribtion, and is only used if `durable_subscriber` +is set to `true`. Please consult your JMS Provider documentation for constraints and requirements for this setting. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, JMS specific properties) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-headers_target>> and +<<{version}-plugins-{type}s-{plugin}-properties_target>>. + +[id="{version}-plugins-{type}s-{plugin}-factory"] +===== `factory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Full name (including package name) of Java connection factory used to create a connection with your JMS provider. + +[id="{version}-plugins-{type}s-{plugin}-factory_settings"] +===== `factory_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Hash of implementation specific configuration values to set on the connection factory of the JMS provider. Each JMS + Provider will have its own set of parameters that can be used here. These parameters are mapped to `set` methods on + the provided connection factory, and can be supplied in either 'snake' or 'camel' case. For example, a hash including + `exclusive_consumer => true` would call `setExclusiveConsumer(true)` on the supplied connection factory. + See your JMS provider documentation for implementation specific details. + +[id="{version}-plugins-{type}s-{plugin}-headers_target"] +===== `headers_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: no default value for this setting + ** ECS Compatibility enabled: `"[@metadata][input][jms][headers]" + +The name of the field under which JMS headers will be added, if <<{version}-plugins-{type}s-{plugin}-include_headers>> is set. + +[id="{version}-plugins-{type}s-{plugin}-include_body"] +===== `include_body` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Include JMS Message Body in the event. +Supports TextMessage, MapMessage and BytesMessage. + +If the JMS Message is a TextMessage or BytesMessage, then the value will be in +the "message" field of the event. If the JMS Message is a MapMessage, then all +the key/value pairs will be added at the top-level of the event by default. +To avoid pollution of the top-level namespace, when receiving a MapMessage, use the <<{version}-plugins-{type}s-{plugin}-target>>. + +StreamMessage and ObjectMessage are not supported. + +[id="{version}-plugins-{type}s-{plugin}-include_header"] +===== `include_header` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * This option is deprecated + +Note: This option is deprecated and it will be removed in the next major version of Logstash. +Use `include_headers` instead. + +[id="{version}-plugins-{type}s-{plugin}-include_headers"] +===== `include_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A JMS message has three parts: + + * Message Headers (required) + * Message Properties (optional) + * Message Body (optional) + +You can tell the input plugin which parts should be included in the event produced by Logstash. + +Include standard JMS message header field values in the event. +Example headers: +[source,ruby] +----- + { + "jms_message_id" => "ID:amqhost-39547-1636977297920-71:1:1:1:1", + "jms_timestamp" => 1636977329102, + "jms_expiration" => 0, + "jms_delivery_mode" => "persistent", + "jms_redelivered" => false, + "jms_destination" => "topic://41ad5342149901ad", + "jms_priority" => 4, + "jms_type" => "sample", + "jms_correlation_id" => "28d975cb-14ff-4285-841e-05ef1e0a7ab2" + } +----- + +[id="{version}-plugins-{type}s-{plugin}-include_properties"] +===== `include_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Include JMS Message Properties Field values in the event. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Polling interval in seconds. +This is the time sleeping between asks to a consumed Queue. +This parameter has non influence in the case of a subcribed Topic. + +[id="{version}-plugins-{type}s-{plugin}-jndi_context"] +===== `jndi_context` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Only used if using JNDI lookup. Key value pairs to determine how to connect the JMS message brokers if JNDI +is being used. Consult your JMS provider documentation for the correct values to use here. + + +[id="{version}-plugins-{type}s-{plugin}-jndi_name"] +===== `jndi_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Only used if using JNDI lookup. Name of JNDI entry at which the Factory can be found. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify it here. This does not work with .pem keys + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password + +[id="{version}-plugins-{type}s-{plugin}-oracle_aq_buffered_messages"] +===== `oracle_aq_buffered_messages` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Receive Oracle AQ buffered messages. +In this mode persistent Oracle AQ JMS messages will not be received. +Only for use with Oracle AQ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Password to use when connecting to the JMS provider. + +[id="{version}-plugins-{type}s-{plugin}-properties_target"] +===== `properties_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: no default value for this setting + ** ECS Compatibility enabled: `"[@metadata][input][jms][properties]" + +The name of the field under which JMS properties will be added, if <<{version}-plugins-{type}s-{plugin}-include_properties>> is set. + +[id="{version}-plugins-{type}s-{plugin}-pub_sub"] +===== `pub_sub` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If pub-sub (topic) style should be used. Note that if `pub_sub` is set to true, `threads` must be set to 1. + +[id="{version}-plugins-{type}s-{plugin}-require_jars"] +===== `require_jars` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +If you do not use an yaml configuration use either the factory or jndi_name. +An optional array of Jar file names to load for the specified +JMS provider. By using this option it is not necessary +to put all the JMS Provider specific jar files into the +java CLASSPATH prior to starting Logstash. + +[id="{version}-plugins-{type}s-{plugin}-runner"] +===== `runner` + + * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions. + + +[id="{version}-plugins-{type}s-{plugin}-selector"] +===== `selector` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JMS message selector. Use in conjunctions with message headers or properties to filter messages to be processed. Only +messages that match the query specified here will be processed. Check with your JMS provider for the correct JMS message +selector syntax. + +[id="{version}-plugins-{type}s-{plugin}-skip_headers"] +===== `skip_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +If `include_headers` is set, a list of headers to skip processing on can be specified here. + +[id="{version}-plugins-{type}s-{plugin}-skip_properties"] +===== `skip_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +If `include_properties` is set, a list of properties to skip processing on can be specified here. + +[id="{version}-plugins-{type}s-{plugin}-system_properties"] +===== `system_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Any System properties that the JMS provider requires can be set either in a Hash here, or in `jvm.options` + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the field to assign MapMessage data into. +If not specified data will be stored in the root of the event. + +NOTE: For TextMessage and BytesMessage the `target` has no effect. Use the codec setting instead +e.g. `codec => json { target => "[jms]" }`. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +NOTE: If pub_sub is set to `true`, this value *must* be `1`. A configuration error will be thrown otherwise! + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Initial connection timeout in seconds. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify it here. This does not work with .pem certs. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. + +* Note, most .jks files created with keytool require a password. + + +[id="{version}-plugins-{type}s-{plugin}-use_jms_timestamp"] +===== `use_jms_timestamp` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Convert the JMSTimestamp header field to the @timestamp value of the event + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use for connecting to JMS provider. + +[id="{version}-plugins-{type}s-{plugin}-yaml_file"] +===== `yaml_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Yaml config file + +[id="{version}-plugins-{type}s-{plugin}-yaml_section"] +===== `yaml_section` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Yaml config file section name +For some known examples, see https://github.com/reidmorrison/jruby-jms/blob/master/examples/jms.yml[jms.yml examples]. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/jms-v3.3.1.asciidoc b/docs/versioned-plugins/inputs/jms-v3.3.1.asciidoc new file mode 100644 index 000000000..45e9e29c5 --- /dev/null +++ b/docs/versioned-plugins/inputs/jms-v3.3.1.asciidoc @@ -0,0 +1,811 @@ +:plugin: jms +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.3.1 +:release_date: 2025-09-08 +:changelog_url: https://github.com/logstash-plugins/logstash-input-jms/blob/v3.3.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jms input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events from a Jms Broker. Supports both Jms Queues and Topics. + +For more information about Jms, see . +For more information about the Ruby Gem used, see . + + +JMS configurations can be done either entirely in the Logstash configuration file, or in a mixture of the Logstash + configuration file, and a specified yaml file. Simple configurations that do not need to make calls to implementation + specific methods on the connection factory can be specified entirely in the Logstash configuration, whereas more complex + configurations, should also use the combination of yaml file and Logstash configuration. + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +JMS data is application specific. ECS compliance for fields depends on the use case. +The plugin includes sensible defaults that change based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>. +When ECS compatibility is disabled, headers, properties, and payload are targeted at the root level of the event to maintain compatibility with legacy usage of this plugin. +When targeting an ECS version, headers and properties target `@metadata` sub-fields unless configured otherwise in order to avoid conflict with ECS fields. +See <<{version}-plugins-{type}s-{plugin}-headers_target>>, <<{version}-plugins-{type}s-{plugin}-properties_target>> and <<{version}-plugins-{type}s-{plugin}-target>>. + +==== Sample Configuration using Logstash Configuration Only + +Configurations can be configured either entirely in Logstash configuration, or via a combination of Logstash configuration +and yaml file, which can be useful for sharing similar configurations across multiple inputs and outputs. +The JMS plugin can also be configured using JNDI if desired. + +===== Logstash Confuguration for Non-JNDI Connection +[source,ruby] +-------------------------------------------------- + input + { + jms { + broker_url => 'failover:(tcp://host1:61616,tcp://host2:61616)?initialReconnectDelay=100' <1> + destination => 'myqueue' <2> + factory => 'org.apache.activemq.ActiveMQConnectionFactory' <3> + pub_sub => false <4> + use_jms_timestamp => false <5> + # JMS provider credentials if needed <6> + username => 'username' + password => 'secret' + # JMS provider keystore and truststore details <7> + keystore => '/Users/logstash-user/security/keystore.jks' + keystore_password => 'another_secret' + truststore => '/Users/logstash-user/security/truststore.jks' + truststore_password => 'yet_another_secret' + # Parts of the JMS message to be included <8> + include_headers => false + include_properties => false + include_body => true + # Message selector + selector => "string_property = 'this' OR int_property < 3" <9> + # Connection factory specific settings + factory_settings => { <10> + exclusive_consumer => true + } + # Jar Files to include + require_jars => ['/usr/share/jms/activemq-all-5.15.9.jar'] <11> + } + } +-------------------------------------------------- + +<1> Url of the broker to connect to. Please consult your JMS provider documentation for the exact syntax to use + here, including how to enable failover. +<2> Name of the topic or queue that the plugin will listen to events from. +<3> Full name (including package name) of Java connection factory used to create a connection with your JMS provider. +<4> Determines whether the event source is a queue or a topic, set to `true` for topic, `false` for queue. +<5> Determines whether the JMSTimestamp header is used to populate the `@timestamp` field. +<6> Credentials to use when connecting to the JMS provider, if required. +<7> Keystore and Truststore to use when connecting to the JMS provider, if required. +<8> Parts of the JMS Message to include in the event - headers, properties and the message body can be included or + excluded from the event. +<9> Message selector: Use this to filter messages to be processed. The whole selector query should be double-quoted, + string property values should be single quoted, and numeric property vaues should not be quoted. + See JMS provider documentation for exact syntax. +<10> Additional settings that are set directly on the ConnectionFactory object can be added here. +<11> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + +===== Logstash Configuration for JNDI Connection +[source,ruby] +-------------------------------------------------- + input { + jms { + # Logstash Configuration Settings. <1> + include_headers => false + include_properties => false + include_body => true + use_jms_timestamp => false + destination => "myqueue" + pub_sub => false + # JNDI Settings + jndi_name => /jms/cf/default <2> + jndi_context => { <3> + 'java.naming.factory.initial' => com.solacesystems.jndi.SolJNDIInitialContextFactory + 'java.naming.security.principal' => solace-cloud-client@user + 'java.naming.provider.url' => tcp://address.messaging.solace.cloud:20608 + 'java.naming.security.credentials' => verysecret + } + # Jar files to be imported + require_jars=> ['/usr/share/jms/commons-lang-2.6.jar', <4> + '/usr/share/jms/sol-jms-10.5.0.jar', + '/usr/share/jms/geronimo-jms_1.1_spec-1.1.1.jar', + '/usr/share/jms/commons-lang-2.6.jar]' + } + } +-------------------------------------------------- + +<1> Configuration settings. Note that there is no `broker_url` or `username` or `password` defined here - these are + defined in the `jndi_context` hash. +<2> JNDI name for this connection. +<3> JNDI context settings hash. Contains details of how to connect to JNDI server. See your JMS provider documentation for + implementation specific details. +<4> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + + +==== Sample Configuration using Logstash Configuration and Yaml File + +===== Non-JNDI Connection + +This section contains sample configurations for connecting to a JMS provider that is not using JNDI using a combination + of the Logstash configuration and the yaml file + +===== Logstash Configuration for Non-JNDI Connection (for configs including yaml) + +[source,ruby] +-------------------------------------------------- + input { + jms { + # Logstash Configuration File Settings <1> + include_headers => false + include_properties => false + include_body => true + use_jms_timestamp => false + destination => "myqueue" + pub_sub => false + # JMS Provider credentials <2> + username => xxxx + password => xxxx + # Location of yaml file, and which section to use for configuration + yaml_file => "~/jms.yml" <3> + yaml_section => "mybroker" <4> + } + } +-------------------------------------------------- + + +<1> Configuration settings +<2> Username and password for the connection. +<3> Full path to a yaml file containing the definition for the ConnectionFactory. +<4> Section name in the yaml file of the ConnectionFactory for this plugin definition + + +===== Yaml File for Non-JNDI Connection + +[source,yaml] +-------------------------------------------------- +mybroker: <1> + :broker_url: 'ssl://localhost:61617' <2> + :factory: org.apache.activemq.ActiveMQConnectionFactory <3> + :exclusive_consumer: true <4> + :require_jars: <5> + - /usr/share/jms/activemq-all-5.15.9.jar + - /usr/share/jms/log4j-1.2.17.jar +-------------------------------------------------- + +<1> Section name for this broker definition. This should be the value of `yaml_section` in the logstash configuration file. + Note that multiple sections can co-exist in the same yaml file. +<2> Full url of the broker. See your JMS Provider documentation for details. +<3> Full name (including package name) of Java connection factory used to create a connection with your JMS provider. +<4> Implementation specific configuration parameters to be used with the connection factory specified. + in <3>. Each JMS Provider will have its own set of parameters that can be used here. These parameters are mapped to + `set` methods on the provided connection factory, and can be supplied in either 'snake' or 'camel' case. In <4> above, + the `exclusive_consumer` property will call the `setExclusiveConsumer` on the supplied connection factory. See your JMS provider + documentation for implementation specific details. +<5> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + + +===== JNDI Connection + +This section contains sample configurations for connecting to a JMS provider that is using JNDI using a combination + of the Logstash configuration and the yaml file + +===== Logstash Configuration for JNDI Connection (for configs including yaml) + +[source,ruby] +-------------------------------------------------- + input { + jms { + # Logstash specific configuration settings <1> + include_headers => false + include_properties => false + include_body => true + use_jms_timestamp => false + destination => "myqueue" + pub_sub => false + # Location of yaml file, and which section to use for configuration + yaml_file => "~/jms.yml" <2> + yaml_section => "mybroker" <3> + } + } +-------------------------------------------------- + +<1> Configuration settings +<2> Full path to a yaml file containing the definition for the ConnectionFactory. +<3> Section name in the yaml file of the ConnectionFactory for this plugin definition + + +===== Yaml File for JNDI Connection + +[source,yaml] +-------------------------------------------------- + +solace: <1> + :jndi_name: /jms/cf/default <2> + :jndi_context: <3> + java.naming.factory.initial: com.solacesystems.jndi.SolJNDIInitialContextFactory + java.naming.security.principal: solace-cloud-client@user + java.naming.provider.url: tcp://address.messaging.solace.cloud:20608 + java.naming.security.credentials: verysecret + :require_jars: <4> + - /usr/share/jms/commons-lang-2.6.jar + - /usr/share/jms/sol-jms-10.5.0.jar + - /usr/share/jms/geronimo-jms_1.1_spec-1.1.1.jar + - /usr/share/jms/commons-lang-2.6.jar +-------------------------------------------------- + +<1> Section name for this broker definition. This should be the value of `yaml_section` in the Logstash configuration file. +<2> Name of JNDI entry at which the Factory can be found +<3> JNDI context settings. Contains details of how to connect to JNDI server. See your JMS provider documentation for + implementation specific details. +<4> List of jars required by the JMS provider. Paths should be the fully qualified location of all jar files required + by the JMS provider. This list may also include dependent jars as well as specific jars from the JMS provider. + + +==== Jar files + +In order to communicate with a JMS broker, the plugin must load the jar files necessary for each client type. +This can be set in the yaml file, or in the main configuration if a yaml file is not necessary. The `require_jars` + setting should include the full path for each jar file required for the client. Eg + +===== Logstash configuration + +[source,ruby] +-------------------------------------------------- + + input { + jms { + : + [snip] + require_jars => ['/usr/share/jms/commons-lang-2.6.jar', + '/usr/share/jms/sol-jms-10.5.0.jar', + '/usr/share/jms/geronimo-jms_1.1_spec-1.1.1.jar', + '/usr/share/jms/commons-lang-2.6.jar'] + } + } +-------------------------------------------------- + +==== Troubleshooting + +This section includes some common issues that a user may have when integrating this plugin with their JMS provider. + +===== Missing Jar files + +The most common issue is missing jar files, which may be jar files provided by the JMS vendor, or jar files that the JMS + vendor requires to run. This issue can manifest in different ways, depending on where the missing jar file is discovered. + +Example log output: + +[source,txt] +----- + Failed to load JMS Connection, likely because a JMS Provider is not on the Logstash classpath or correctly + specified by the plugin's `require_jars` directive + {:exception=>"cannot load Java class javax.jms.DeliveryMode" +----- + +[source,txt] +----- + JMS Consumer Died {:exception=>"Java::JavaxNaming::NoInitialContextException", + :exception_message=>"Cannot instantiate class:" +----- + +[source,txt] +----- + warning: thread "[main]"Java::JavaxJms::JMSException", :exception_message=>"io/netty/channel/epoll/Epoll", + :root_cause=>{:exception=>"Java::JavaLang::NoClassDefFoundError", :exception_message=>"io/netty/channel/epoll/Epoll"} +----- + +If any of these issues occur, check the list of `require_jars` in either the Logstash configuration or yaml configuration + files. + +===== Setting System Properties + +Many JMS providers allow or expect System properties to be set to configure certain properties when using JMS, for +example, the Apache qpid JMS client allows the connection factory lookup to be stored there, and the Solace JMS client +allows many properties, such as number of connection retries to be set as System properties. Any system properties that +are set should be set in the Logstash `jvm.options` file. + +===== Multiple JMS inputs/outputs in the same Logstash process + +The use of multiple JMS consumers and producers in the same Logstash process is unsupported if: + + * System properties need to be different for any of the consumers/producers + * Different keystores or truststores are required for any of the consumers/producers + +===== Message Selectors unexpectedly filtering out all messages + +Incorrect message selector syntax can have two effects - either the syntax is incorrect and the selector parser from the +JMS provider will throw an exception causing the plugin to fail OR the syntax will be accepted, but the messages will +be silently dropped - this can happen with incorrect quoting of string properties in the selector definition. All +selector definitions must be double quoted in the Logstash configuration file, and string property values must be +single quoted, and numeric property values not quoted at all. + + +===== Failed to create Event with MissingConverterException + +Messages from certain JMS providers may contain headers or properties that Logstash cannot interpret, which can lead to +error messages such as: + + +[source,txt] +----- +[2019-11-25T08:04:28,769][ERROR][logstash.inputs.jms ] Failed to create event {:message=>Java::ComSolacesystemsJmsMessage::SolTextMessage: ... +Attributes: {:jms_correlation_id=>"xxxx", :jms_delivery_mode_sym=>:non_persistent, :jms_destination=>"destination", :jms_expiration=>0, :jms_message_id=>"xxxxxx", :jms_priority=>0, :jms_redelivered=>false, :jms_reply_to=>#, :jms_timestamp=>1574669008862, :jms_type=>nil} +Properties: nil, :exception=>org.logstash.MissingConverterException: Missing Converter handling for full class name=com.solacesystems.jms.impl.SolTopicImpl, simple name=SolTopicImpl, :backtrace=>["org.logstash.Valuefier.fallbackConvert(Valuefier.java:98)..."]} +:exception=>org.logstash.MissingConverterException: Missing Converter handling for full class name=com.solacesystems.jms.impl.SolTopicImpl +----- + +To get around this, use the `skip_headers` or `skip_properties` configuration setting to avoid attempting to process +the offending header or property in the message. + +In the example shown above, this attribute is causing the +`MissingConverterException`: + +`jms_reply_to=>#` + + +To avoid this error, the configuration should include the following line: + +[source,ruby] +----- +skip_headers => ["jms_reply_to"] +----- + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jms Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-broker_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-destination>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-durable_subscriber>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-durable_subscriber_client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-durable_subscriber_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-factory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-factory_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-headers_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_body>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_header>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_headers>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_properties>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jndi_context>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jndi_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-oracle_aq_buffered_messages>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-properties_target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pub_sub>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-require_jars>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-runner>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-selector>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-skip_headers>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-skip_properties>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-system_properties>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-use_jms_timestamp>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-yaml_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-yaml_section>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-broker_url"] +===== `broker_url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Url to use when connecting to the JMS provider. This is only relevant for non-JNDI configurations. + +[id="{version}-plugins-{type}s-{plugin}-destination"] +===== `destination` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the destination queue or topic to use. + +TIP: If the destination setting doesn't appear to be working properly, try this format: `!`. + +[id="{version}-plugins-{type}s-{plugin}-durable_subscriber"] +===== `durable_subscriber` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * This is `false` by default + * Requires `pub_sub` to be true + +Setting this value to `true` will make subscriptions to topics "durable", which allowing messages that arrived on the +specified topic while Logstash is not running to still be read. Without setting this value, any messages sent to a topic +while Logstash is not actively listening will be lost. A durable subscriber specifies a unique identity consisting of the + topic (`destination`), the client id (`durable_subscriber_client_id`) and subscriber name + (`durable_subscriber_subscriber_name`). See your JMS Provider documentation for any further requirements/limitations + around these settings. + +* Note that a durable subscription can only have one active subscriber at a time. +* Note that this setting is only permitted when `pub_sub` is set to true, and will generate a configuration error otherwise + +[id="{version}-plugins-{type}s-{plugin}-durable_subscriber_client_id"] +===== `durable_subscriber_client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * If `durable_subscriber` is set, the default value for this setting is 'Logstash', otherwise this setting has no + effect + +This represents the value of the client ID for a durable subscribtion, and is only used if `durable_subscriber` is set to `true`. + +[id="{version}-plugins-{type}s-{plugin}-durable_subscriber_name"] +===== `durable_subscriber_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * If `durable_subscriber` is set, the default value for this setting will be the same value as the `destination` + setting, otherwise this setting has no effect. + +This represents the value of the subscriber name for a durable subscribtion, and is only used if `durable_subscriber` +is set to `true`. Please consult your JMS Provider documentation for constraints and requirements for this setting. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, JMS specific properties) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <<{version}-plugins-{type}s-{plugin}-headers_target>> and +<<{version}-plugins-{type}s-{plugin}-properties_target>>. + +[id="{version}-plugins-{type}s-{plugin}-factory"] +===== `factory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Full name (including package name) of Java connection factory used to create a connection with your JMS provider. + +[id="{version}-plugins-{type}s-{plugin}-factory_settings"] +===== `factory_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Hash of implementation specific configuration values to set on the connection factory of the JMS provider. Each JMS + Provider will have its own set of parameters that can be used here. These parameters are mapped to `set` methods on + the provided connection factory, and can be supplied in either 'snake' or 'camel' case. For example, a hash including + `exclusive_consumer => true` would call `setExclusiveConsumer(true)` on the supplied connection factory. + See your JMS provider documentation for implementation specific details. + +[id="{version}-plugins-{type}s-{plugin}-headers_target"] +===== `headers_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: no default value for this setting + ** ECS Compatibility enabled: `"[@metadata][input][jms][headers]" + +The name of the field under which JMS headers will be added, if <<{version}-plugins-{type}s-{plugin}-include_headers>> is set. + +[id="{version}-plugins-{type}s-{plugin}-include_body"] +===== `include_body` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Include JMS Message Body in the event. +Supports TextMessage, MapMessage and BytesMessage. + +If the JMS Message is a TextMessage or BytesMessage, then the value will be in +the "message" field of the event. If the JMS Message is a MapMessage, then all +the key/value pairs will be added at the top-level of the event by default. +To avoid pollution of the top-level namespace, when receiving a MapMessage, use the <<{version}-plugins-{type}s-{plugin}-target>>. + +StreamMessage and ObjectMessage are not supported. + +[id="{version}-plugins-{type}s-{plugin}-include_header"] +===== `include_header` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * This option is deprecated + +Note: This option is deprecated and it will be removed in the next major version of Logstash. +Use `include_headers` instead. + +[id="{version}-plugins-{type}s-{plugin}-include_headers"] +===== `include_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A JMS message has three parts: + + * Message Headers (required) + * Message Properties (optional) + * Message Body (optional) + +You can tell the input plugin which parts should be included in the event produced by Logstash. + +Include standard JMS message header field values in the event. +Example headers: +[source,ruby] +----- + { + "jms_message_id" => "ID:amqhost-39547-1636977297920-71:1:1:1:1", + "jms_timestamp" => 1636977329102, + "jms_expiration" => 0, + "jms_delivery_mode" => "persistent", + "jms_redelivered" => false, + "jms_destination" => "topic://41ad5342149901ad", + "jms_priority" => 4, + "jms_type" => "sample", + "jms_correlation_id" => "28d975cb-14ff-4285-841e-05ef1e0a7ab2" + } +----- + +[id="{version}-plugins-{type}s-{plugin}-include_properties"] +===== `include_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Include JMS Message Properties Field values in the event. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Polling interval in seconds. +This is the time sleeping between asks to a consumed Queue. +This parameter has non influence in the case of a subcribed Topic. + +[id="{version}-plugins-{type}s-{plugin}-jndi_context"] +===== `jndi_context` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Only used if using JNDI lookup. Key value pairs to determine how to connect the JMS message brokers if JNDI +is being used. Consult your JMS provider documentation for the correct values to use here. + + +[id="{version}-plugins-{type}s-{plugin}-jndi_name"] +===== `jndi_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Only used if using JNDI lookup. Name of JNDI entry at which the Factory can be found. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify it here. This does not work with .pem keys + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password + +[id="{version}-plugins-{type}s-{plugin}-oracle_aq_buffered_messages"] +===== `oracle_aq_buffered_messages` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Receive Oracle AQ buffered messages. +In this mode persistent Oracle AQ JMS messages will not be received. +Only for use with Oracle AQ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Password to use when connecting to the JMS provider. + +[id="{version}-plugins-{type}s-{plugin}-properties_target"] +===== `properties_target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: no default value for this setting + ** ECS Compatibility enabled: `"[@metadata][input][jms][properties]" + +The name of the field under which JMS properties will be added, if <<{version}-plugins-{type}s-{plugin}-include_properties>> is set. + +[id="{version}-plugins-{type}s-{plugin}-pub_sub"] +===== `pub_sub` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If pub-sub (topic) style should be used. Note that if `pub_sub` is set to true, `threads` must be set to 1. + +[id="{version}-plugins-{type}s-{plugin}-require_jars"] +===== `require_jars` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +If you do not use an yaml configuration use either the factory or jndi_name. +An optional array of Jar file names to load for the specified +JMS provider. By using this option it is not necessary +to put all the JMS Provider specific jar files into the +java CLASSPATH prior to starting Logstash. + +[id="{version}-plugins-{type}s-{plugin}-runner"] +===== `runner` + + * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions. + + +[id="{version}-plugins-{type}s-{plugin}-selector"] +===== `selector` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JMS message selector. Use in conjunctions with message headers or properties to filter messages to be processed. Only +messages that match the query specified here will be processed. Check with your JMS provider for the correct JMS message +selector syntax. + +[id="{version}-plugins-{type}s-{plugin}-skip_headers"] +===== `skip_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +If `include_headers` is set, a list of headers to skip processing on can be specified here. + +[id="{version}-plugins-{type}s-{plugin}-skip_properties"] +===== `skip_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +If `include_properties` is set, a list of properties to skip processing on can be specified here. + +[id="{version}-plugins-{type}s-{plugin}-system_properties"] +===== `system_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Any System properties that the JMS provider requires can be set either in a Hash here, or in `jvm.options` + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the field to assign MapMessage data into. +If not specified data will be stored in the root of the event. + +NOTE: For TextMessage and BytesMessage the `target` has no effect. Use the codec setting instead +e.g. `codec => json { target => "[jms]" }`. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +NOTE: If pub_sub is set to `true`, this value *must* be `1`. A configuration error will be thrown otherwise! + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Initial connection timeout in seconds. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify it here. This does not work with .pem certs. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. + +* Note, most .jks files created with keytool require a password. + + +[id="{version}-plugins-{type}s-{plugin}-use_jms_timestamp"] +===== `use_jms_timestamp` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Convert the JMSTimestamp header field to the @timestamp value of the event + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to use for connecting to JMS provider. + +[id="{version}-plugins-{type}s-{plugin}-yaml_file"] +===== `yaml_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Yaml config file + +[id="{version}-plugins-{type}s-{plugin}-yaml_section"] +===== `yaml_section` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Yaml config file section name +For some known examples, see https://github.com/reidmorrison/jruby-jms/blob/master/examples/jms.yml[jms.yml examples]. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-index.asciidoc b/docs/versioned-plugins/inputs/kafka-index.asciidoc index 8da055a08..869248630 100644 --- a/docs/versioned-plugins/inputs/kafka-index.asciidoc +++ b/docs/versioned-plugins/inputs/kafka-index.asciidoc @@ -5,6 +5,21 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-08-28 +| <> | 2025-06-12 +| <> | 2025-05-21 +| <> | 2025-05-08 +| <> | 2025-01-07 +| <> | 2024-12-18 +| <> | 2024-11-25 +| <> | 2024-10-04 +| <> | 2024-07-26 +| <> | 2024-07-02 +| <> | 2024-05-17 +| <> | 2024-04-10 +| <> | 2024-04-09 +| <> | 2024-02-06 +| <> | 2024-01-09 | <> | 2023-10-16 | <> | 2023-10-04 | <> | 2023-07-04 @@ -12,6 +27,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-02-14 | <> | 2023-01-25 | <> | 2022-12-16 +| <> | 2025-02-12 | <> | 2023-10-16 | <> | 2022-05-09 | <> | 2022-04-25 @@ -59,6 +75,21 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-11 |======================================================================= +include::kafka-v11.6.4.asciidoc[] +include::kafka-v11.6.3.asciidoc[] +include::kafka-v11.6.2.asciidoc[] +include::kafka-v11.6.1.asciidoc[] +include::kafka-v11.6.0.asciidoc[] +include::kafka-v11.5.4.asciidoc[] +include::kafka-v11.5.3.asciidoc[] +include::kafka-v11.5.2.asciidoc[] +include::kafka-v11.5.1.asciidoc[] +include::kafka-v11.5.0.asciidoc[] +include::kafka-v11.4.2.asciidoc[] +include::kafka-v11.4.1.asciidoc[] +include::kafka-v11.4.0.asciidoc[] +include::kafka-v11.3.4.asciidoc[] +include::kafka-v11.3.3.asciidoc[] include::kafka-v11.3.2.asciidoc[] include::kafka-v11.3.1.asciidoc[] include::kafka-v11.3.0.asciidoc[] @@ -66,6 +97,7 @@ include::kafka-v11.2.1.asciidoc[] include::kafka-v11.2.0.asciidoc[] include::kafka-v11.1.0.asciidoc[] include::kafka-v11.0.0.asciidoc[] +include::kafka-v10.12.2.asciidoc[] include::kafka-v10.12.1.asciidoc[] include::kafka-v10.12.0.asciidoc[] include::kafka-v10.11.0.asciidoc[] diff --git a/docs/versioned-plugins/inputs/kafka-v10.12.2.asciidoc b/docs/versioned-plugins/inputs/kafka-v10.12.2.asciidoc new file mode 100644 index 000000000..ad724cc20 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v10.12.2.asciidoc @@ -0,0 +1,719 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 2.8 +:kafka_client_doc: 25 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v10.12.2 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v10.12.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regex pattern to subscribe to. +The topics configuration will be ignored when using this configuration. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.3.3.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.3.3.asciidoc new file mode 100644 index 000000000..6ae6222b8 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.3.3.asciidoc @@ -0,0 +1,807 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.3.3 +:release_date: 2024-01-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.3.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.3.4.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.3.4.asciidoc new file mode 100644 index 000000000..5671ea33c --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.3.4.asciidoc @@ -0,0 +1,807 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.3.4 +:release_date: 2024-02-06 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.3.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.4.0.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.4.0.asciidoc new file mode 100644 index 000000000..3a36708cf --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.4.0.asciidoc @@ -0,0 +1,807 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.0 +:release_date: 2024-04-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.4.1.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.4.1.asciidoc new file mode 100644 index 000000000..396bd3b81 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.4.1.asciidoc @@ -0,0 +1,807 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.1 +:release_date: 2024-04-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.4.2.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.4.2.asciidoc new file mode 100644 index 000000000..54715e17c --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.4.2.asciidoc @@ -0,0 +1,807 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.2 +:release_date: 2024-05-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.5.0.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.5.0.asciidoc new file mode 100644 index 000000000..112ad8584 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.5.0.asciidoc @@ -0,0 +1,818 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.0 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.5.1.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.5.1.asciidoc new file mode 100644 index 000000000..9366bb224 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.5.1.asciidoc @@ -0,0 +1,818 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.1 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.5.2.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.5.2.asciidoc new file mode 100644 index 000000000..facbe1cf7 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.5.2.asciidoc @@ -0,0 +1,826 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.2 +:release_date: 2024-10-04 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class""] +===== `sasl_client_callback_handler_class` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.5.3.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.5.3.asciidoc new file mode 100644 index 000000000..5e5fa8772 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.5.3.asciidoc @@ -0,0 +1,826 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.7 +:kafka_client_doc: 37 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.3 +:release_date: 2024-11-25 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class""] +===== `sasl_client_callback_handler_class` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.5.4.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.5.4.asciidoc new file mode 100644 index 000000000..d7e3547e0 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.5.4.asciidoc @@ -0,0 +1,826 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.4 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class""] +===== `sasl_client_callback_handler_class` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.6.0.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.6.0.asciidoc new file mode 100644 index 000000000..4f3e0bcc5 --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.6.0.asciidoc @@ -0,0 +1,882 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.0 +:release_date: 2025-01-07 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.6.1.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.6.1.asciidoc new file mode 100644 index 000000000..906cd826e --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.6.1.asciidoc @@ -0,0 +1,909 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.1 +:release_date: 2025-05-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is <> +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.6.2.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.6.2.asciidoc new file mode 100644 index 000000000..2ec7c838f --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.6.2.asciidoc @@ -0,0 +1,909 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.2 +:release_date: 2025-05-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.6.3.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.6.3.asciidoc new file mode 100644 index 000000000..72c871dfb --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.6.3.asciidoc @@ -0,0 +1,909 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.9.1 +:kafka_client_doc: 39 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.3 +:release_date: 2025-06-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/kafka-v11.6.4.asciidoc b/docs/versioned-plugins/inputs/kafka-v11.6.4.asciidoc new file mode 100644 index 000000000..4fcc7a3df --- /dev/null +++ b/docs/versioned-plugins/inputs/kafka-v11.6.4.asciidoc @@ -0,0 +1,909 @@ +:integration: kafka +:plugin: kafka +:type: input +:default_codec: plain +:kafka_client: 3.9.1 +:kafka_client_doc: 39 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.4 +:release_date: 2025-08-28 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This input will read events from a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This input supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +[NOTE] +======= +This plugin does not support using a proxy when communicating to the Kafka broker. + +This plugin does support using a proxy when communicating to the Schema Registry using the <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> option. +======= + +The Logstash Kafka consumer handles group management and uses the default offset management +strategy using Kafka topics. + +Logstash instances by default form a single logical group to subscribe to Kafka topics +Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, +you could run multiple Logstash instances with the same `group_id` to spread the load across +physical machines. Messages in a topic will be distributed to all Logstash instances with +the same `group_id`. + +Ideally you should have as many threads as the number of partitions for a perfect balance -- +more threads than partitions means that some threads will be idle + +For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer + +Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +==== Metadata fields + +The following metadata from Kafka broker are added under the `[@metadata]` field: + +* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed. +* `[@metadata][kafka][consumer_group]`: Consumer group +* `[@metadata][kafka][partition]`: Partition info for this message. +* `[@metadata][kafka][offset]`: Original record offset for this message. +* `[@metadata][kafka][key]`: Record key, if any. +* `[@metadata][kafka][timestamp]`: Timestamp in the Record. +Depending on your broker configuration, this can be +either when the record was created (default) or when it was received by the +broker. See more about property log.message.timestamp.type at +https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs + +Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`). + +Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be +inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Input Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's consumer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-auto_create_topics>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-auto_offset_reset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-check_crcs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_rack>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-consumer_threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-decorate_events>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-enable_auto_commit>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_internal_topics>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fetch_min_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-group_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-group_instance_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-isolation_level>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_interval_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_poll_records>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partition_assignment_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-poll_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_secret>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_url>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-schema_registry_validation>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-session_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["jks", "PKCS12"]`|No +| <<{version}-plugins-{type}s-{plugin}-topics>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-topics_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-auto_commit_interval_ms"] +===== `auto_commit_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5000`. + +The frequency in milliseconds that the consumer offsets are committed to Kafka. + +[id="{version}-plugins-{type}s-{plugin}-auto_offset_reset"] +===== `auto_offset_reset` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +What to do when there is no initial offset in Kafka or if an offset is out of range: + +* earliest: automatically reset the offset to the earliest offset +* latest: automatically reset the offset to the latest offset +* none: throw exception to the consumer if no previous offset is found for the consumer's group +* anything else: throw exception to the consumer. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +A list of URLs of Kafka instances to use for establishing the initial connection to the cluster. +This list should be in the form of `host1:port1,host2:port2` These urls are just used +for the initial connection to discover the full cluster membership (which may change dynamically) +so this list need not contain the full set of servers (you may want more than one, though, in +case a server is down). + +[id="{version}-plugins-{type}s-{plugin}-check_crcs"] +===== `check_crcs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Automatically check the CRC32 of the records consumed. +This ensures no on-the-wire or on-disk corruption to the messages occurred. +This check adds some overhead, so it may be disabled in cases seeking extreme performance. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"default"` + +How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple +IP addresses for a hostname, they will all be attempted to connect to before failing the +connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. The purpose of this +is to be able to track the source of requests beyond just ip/port by allowing +a logical application name to be included. + +[id="{version}-plugins-{type}s-{plugin}-client_rack"] +===== `client_rack` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A rack identifier for the Kafka consumer. +Used to select the physically closest rack for the consumer to read from. +The setting corresponds with Kafka's `broker.rack` configuration. + +NOTE: Available only for Kafka 2.4.0 and higher. See +https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392]. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-consumer_threads"] +===== `consumer_threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Ideally you should have as many threads as the number of partitions for a perfect +balance — more threads than partitions means that some threads will be idle + +[id="{version}-plugins-{type}s-{plugin}-decorate_events"] +===== `decorate_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: record's attributes are added + - `extended`: record's attributes, headers are added (limited to headers with values using UTF-8 encoding) + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Option to add Kafka metadata like topic, message size and header key values to the event. +This will add a field named `kafka` to the logstash event containing the following attributes: + +* `topic`: The topic this message is associated with +* `consumer_group`: The consumer group used to read in this event +* `partition`: The partition this message is associated with +* `offset`: The offset from the partition this message is associated with +* `key`: A ByteBuffer containing the message key + + +[id="{version}-plugins-{type}s-{plugin}-auto_create_topics"] +===== `auto_create_topics`  + +  * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Controls whether the topic is automatically created when subscribing to a non-existent topic. +A topic will be auto-created only if this configuration is set to `true` and auto-topic creation is enabled on the broker using `auto.create.topics.enable`; +otherwise auto-topic creation is not permitted.  + +[id="{version}-plugins-{type}s-{plugin}-enable_auto_commit"] +===== `enable_auto_commit` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This committed offset will be used when the process fails as the position from +which the consumption will begin. + +If true, periodically commit to Kafka the offsets of messages already returned by +the consumer. If value is `false` however, the offset is committed every time the +consumer writes data fetched from the topic to the in-memory or persistent queue. + +[id="{version}-plugins-{type}s-{plugin}-exclude_internal_topics"] +===== `exclude_internal_topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Whether records from internal topics (such as offsets) should be exposed to the consumer. +If set to true the only way to receive records from an internal topic is subscribing to it. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_bytes"] +===== `fetch_max_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `52428800` (50MB) + +The maximum amount of data the server should return for a fetch request. This is not an +absolute maximum, if the first message in the first non-empty partition of the fetch is larger +than this value, the message will still be returned to ensure that the consumer can make progress. + +[id="{version}-plugins-{type}s-{plugin}-fetch_max_wait_ms"] +===== `fetch_max_wait_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` milliseconds. + +The maximum amount of time the server will block before answering the fetch request if +there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This +should be less than or equal to the timeout used in `poll_timeout_ms` + +[id="{version}-plugins-{type}s-{plugin}-fetch_min_bytes"] +===== `fetch_min_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The minimum amount of data the server should return for a fetch request. If insufficient +data is available the request will wait for that much data to accumulate +before answering the request. + +[id="{version}-plugins-{type}s-{plugin}-group_id"] +===== `group_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber +that happens to be made up of multiple processors. Messages in a topic will be distributed to all +Logstash instances with the same `group_id`. + +NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics, +it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...` +is also recommended. + +[id="{version}-plugins-{type}s-{plugin}-group_instance_id"] +===== `group_instance_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in +https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345], +available under Kafka property `group.instance.id`. +Its purpose is to avoid rebalances in situations in which a lot of data +has to be forwarded after a consumer goes offline. +This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance +A to B would cause a huge amount of data to be transferred. +A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option. + +NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<{version}-plugins-{type}s-{plugin}-group_id>>. +Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected. +You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application. + +NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to +the `group_instance_id` to avoid collisions. + +[id="{version}-plugins-{type}s-{plugin}-heartbeat_interval_ms"] +===== `heartbeat_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3000` milliseconds (3 seconds). + +The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure +that the consumer's session stays active and to facilitate rebalancing when new +consumers join or leave the group. The value must be set lower than +`session.timeout.ms`, but typically should be set no higher than 1/3 of that value. +It can be adjusted even lower to control the expected time for normal rebalances. + +[id="{version}-plugins-{type}s-{plugin}-isolation_level"] +===== `isolation_level` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"read_uncommitted"` + +Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return +transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will +return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned +unconditionally in either mode. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_deserializer_class"] +===== `key_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's key + +[id="{version}-plugins-{type}s-{plugin}-max_partition_fetch_bytes"] +===== `max_partition_fetch_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum amount of data per-partition the server will return. The maximum total memory used for a +request will be `#partitions * max.partition.fetch.bytes`. This size must be at least +as large as the maximum message size the server allows or else it is possible for the producer to +send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying +to fetch a large message on a certain partition. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_interval_ms"] +===== `max_poll_interval_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The maximum delay between invocations of poll() when using consumer group management. This places +an upper bound on the amount of time that the consumer can be idle before fetching more records. +If poll() is not called before expiration of this timeout, then the consumer is considered failed and +the group will rebalance in order to reassign the partitions to another member. + +[id="{version}-plugins-{type}s-{plugin}-max_poll_records"] +===== `max_poll_records` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500`. + +The maximum number of records returned in a single call to poll(). + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The period of time in milliseconds after which we force a refresh of metadata even if +we haven't seen any partition leadership changes to proactively discover any new brokers or partitions + +[id="{version}-plugins-{type}s-{plugin}-partition_assignment_strategy"] +===== `partition_assignment_strategy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the partition assignment strategy that the client uses to distribute +partition ownership amongst consumer instances, supported options are: + +* `range` +* `round_robin` +* `sticky` +* `cooperative_sticky` + +These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`] +implementations. + +[id="{version}-plugins-{type}s-{plugin}-poll_timeout_ms"] +===== `poll_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +Time Kafka consumer will wait to receive new messages from topics. + +After subscribing to a set of topics, the Kafka consumer automatically joins the group when polling. +The plugin poll-ing in a loop ensures consumer liveness. +Underneath the covers, Kafka client sends periodic heartbeats to the server. +The timeout specified the time to block waiting for input on each poll. + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` milliseconds. + +The amount of time to wait before attempting to reconnect to a given host. +This avoids repeatedly connecting to a host in a tight loop. +This backoff applies to all requests sent by the consumer to the broker. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed fetch request +to a given topic partition. This avoids repeated fetching-and-failing in a tight loop. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + input { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_key"] +===== `schema_registry_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the username for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_proxy"] +===== `schema_registry_proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_secret"] +===== `schema_registry_secret` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Set the password for basic authorization to access remote Schema Registry. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"] +===== `schema_registry_ssl_keystore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +If schema registry client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"] +===== `schema_registry_ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +If schema registry authentication is required, this setting stores the keystore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"] +===== `schema_registry_ssl_keystore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"] +===== `schema_registry_ssl_truststore_location` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore path to validate the schema registry's certificate. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"] +===== `schema_registry_ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The schema registry truststore password. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"] +===== `schema_registry_ssl_truststore_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_url"] +===== `schema_registry_url` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + +The URI that points to an instance of the +https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service, +used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from +the specified topics have been uploaded to the Schema Registry service. +The schemas must follow a naming convention with the pattern -value. + +Use either the Schema Registry config option or the +<<{version}-plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-schema_registry_validation"] +===== `schema_registry_validation` + + * Value can be either of: `auto`, `skip` + * Default value is `"auto"` + +NOTE: Under most circumstances, the default setting of `auto` should not need to be changed. + +When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed. +In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash. +This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer (SO_SNDBUF) to use when sending data + +[id="{version}-plugins-{type}s-{plugin}-session_timeout_ms"] +===== `session_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds (10 seconds). + +The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead +and a rebalance operation is triggered for the group identified by `group_id` + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable endpoint verification + + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the keystore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The format of the truststore file. It must be either `jks` or `PKCS12`. + +[id="{version}-plugins-{type}s-{plugin}-topics"] +===== `topics` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["logstash"]` + +A list of topics to subscribe to, defaults to ["logstash"]. + +[id="{version}-plugins-{type}s-{plugin}-topics_pattern"] +===== `topics_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A topic regular expression pattern to subscribe to. + +Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers. + +NOTE: When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription +happens but on the broker side it is logged that the subscription of some topics was denied to the configured user. + +[id="{version}-plugins-{type}s-{plugin}-value_deserializer_class"] +===== `value_deserializer_class` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"` + +Java Class used to deserialize the record's value. +A custom value deserializer can be used only if you are not using a Schema Registry. +Use either the value_deserializer_class config option or the +<<{version}-plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/logstash-index.asciidoc b/docs/versioned-plugins/inputs/logstash-index.asciidoc index e14f7f70e..1f41b281a 100644 --- a/docs/versioned-plugins/inputs/logstash-index.asciidoc +++ b/docs/versioned-plugins/inputs/logstash-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-12-10 +| <> | 2024-09-12 +| <> | 2024-01-23 | <> | 2023-11-16 | <> | 2023-11-06 | <> | 2023-10-03 @@ -14,6 +17,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-09-25 |======================================================================= +include::logstash-v1.0.4.asciidoc[] +include::logstash-v1.0.3.asciidoc[] +include::logstash-v1.0.2.asciidoc[] include::logstash-v1.0.1.asciidoc[] include::logstash-v1.0.0.asciidoc[] include::logstash-v0.0.5.asciidoc[] diff --git a/docs/versioned-plugins/inputs/logstash-v1.0.2.asciidoc b/docs/versioned-plugins/inputs/logstash-v1.0.2.asciidoc new file mode 100644 index 000000000..08b133f5e --- /dev/null +++ b/docs/versioned-plugins/inputs/logstash-v1.0.2.asciidoc @@ -0,0 +1,250 @@ +:integration: logstash +:plugin: logstash +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.2 +:release_date: 2024-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Listen for events that are sent by a {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin] in a pipeline that may be in another process or on another host. +The upstream output must have a TCP route to the port (defaults to 9800) on an interface that this plugin is bound to. + +NOTE: Sending events to this input by _any_ means other than `plugins-outputs-logstash` is neither advised nor supported. +We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding. + +[id="{version}-plugins-{type}s-{plugin}-minimum-config"] +===== Minimum Configuration +[cols="3a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +input { + logstash { + ssl_keystore_path + => "/path/to/logstash.p12" + ssl_keystore_password + => "${PASS}" + } +} +---- + +| + +[source,shell] +---- +input { + logstash { + ssl_enabled => false + } +} +---- + +|======================================================================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-config-binding"] +===== Configuration Concepts + +This input plugin needs to be configured to bind to a TCP <<{version}-plugins-{type}s-{plugin}-port>>, and can be constrained to bind to a particular interface by providing the IP to <<{version}-plugins-{type}s-{plugin}-host>>. + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-identity"] +===== Security: SSL Identity + +Unless SSL is disabled, this plugin needs to be configured with identity material: + +* JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) +* PKCS8-formatted Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-trust"] +===== Security: SSL Trust + +When communicating over SSL, this plugin can be configured to either request or require that connecting clients present their own identity claims with <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>. + +Certificates that are presented by clients are validated by default using the system trust store to ensure that they are currently-valid and trusted, and that the client can prove possession of its associated private key. +You can provide an _alternate_ source of trust with: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) + +NOTE: Client-certificate verification does _not_ verify identity claims on the presented certificate, such as whether the certificate includes a Subject Alt Name matching the IP address from which the client is connecting. + +[id="{version}-plugins-{type}s-{plugin}-config-credentials"] +===== Security: Credentials + +You can also configure this plugin to require a specific username/password be provided by configuring <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>. +Doing so requires connecting `logstash-output` plugin clients to provide matching `username` and `password`. + +NOTE: when SSL is disabled, data and credentials will be received in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Logstash Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type |Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string] |No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number] |No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + +* Value type is a {logstash-ref}/configuration-file-structure.html#string[string] ip address +* Default value is `0.0.0.0` (all interfaces) + +Specify which interface to listen on by providing its ip address. +By default, this input listens on all available interfaces. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Password for password-based authentication. +Requires <<{version}-plugins-{type}s-{plugin}-username>>. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + +* Value type is a {logstash-ref}/configuration-file-structure.html#number[number] port +* Default value is 9800 + +Specify which port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* Cannot be combined with configurations that disable SSL. + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting clients. +The certificate _SHOULD_ include identity claims about the ip address or hostname that clients use to establish a connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication, `+ssl_client_authentication => none+`>>. + +One or more PEM-encoded files defining certificate authorities for use in client authentication. +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: +** `none`: do not request client's certificate, or validate certificates that are presented +** `optional`: request client's certificate, and validate it against our trust authorities _if-and-only-if_ it is presented +** `required`: require a valid certificate from the client that is signed by a trusted certificate authority +* Default value is `"none"` + +By default the server doesn't do any client authentication. +This means that connections from clients are _private_ when SSL is enabled, but that this input will allow SSL connections from _any_ client. +If you wish to configure this plugin to reject connections from untrusted hosts, you will need to configure this plugin to authenticate clients, and may also need to configure its <<{version}-plugins-{type}s-{plugin}-config-ssl-trust, source of trust>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +SSL is enabled by default, which requires configuring this plugin to present its <<{version}-plugins-{type}s-{plugin}-config-ssl-identity, identity>>. + +You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. +* Cannot be combined with configurations that disable SSL. + +A path to a PEM-encoded _encrypted_ PKCS8 SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* Cannot be combined with configurations that disable SSL. + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. +* Cannot be combined with configurations that disable SSL. + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Username for password-based authentication. +When this input plugin is configured with a `username`, it also requires a `password`, and any upstream `logstash-output` plugin must also be configured with a matching `username`/`password` pair. + +NOTE: when SSL is disabled, credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/logstash-v1.0.3.asciidoc b/docs/versioned-plugins/inputs/logstash-v1.0.3.asciidoc new file mode 100644 index 000000000..d2a0d55b3 --- /dev/null +++ b/docs/versioned-plugins/inputs/logstash-v1.0.3.asciidoc @@ -0,0 +1,250 @@ +:integration: logstash +:plugin: logstash +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.3 +:release_date: 2024-09-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Listen for events that are sent by a {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin] in a pipeline that may be in another process or on another host. +The upstream output must have a TCP route to the port (defaults to 9800) on an interface that this plugin is bound to. + +NOTE: Sending events to this input by _any_ means other than `plugins-outputs-logstash` is neither advised nor supported. +We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding. + +[id="{version}-plugins-{type}s-{plugin}-minimum-config"] +===== Minimum Configuration +[cols="3a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +input { + logstash { + ssl_keystore_path + => "/path/to/logstash.p12" + ssl_keystore_password + => "${PASS}" + } +} +---- + +| + +[source,shell] +---- +input { + logstash { + ssl_enabled => false + } +} +---- + +|======================================================================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-config-binding"] +===== Configuration Concepts + +This input plugin needs to be configured to bind to a TCP <<{version}-plugins-{type}s-{plugin}-port>>, and can be constrained to bind to a particular interface by providing the IP to <<{version}-plugins-{type}s-{plugin}-host>>. + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-identity"] +===== Security: SSL Identity + +Unless SSL is disabled, this plugin needs to be configured with identity material: + +* JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) +* PKCS8-formatted Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-trust"] +===== Security: SSL Trust + +When communicating over SSL, this plugin can be configured to either request or require that connecting clients present their own identity claims with <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>. + +Certificates that are presented by clients are validated by default using the system trust store to ensure that they are currently-valid and trusted, and that the client can prove possession of its associated private key. +You can provide an _alternate_ source of trust with: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) + +NOTE: Client-certificate verification does _not_ verify identity claims on the presented certificate, such as whether the certificate includes a Subject Alt Name matching the IP address from which the client is connecting. + +[id="{version}-plugins-{type}s-{plugin}-config-credentials"] +===== Security: Credentials + +You can also configure this plugin to require a specific username/password be provided by configuring <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>. +Doing so requires connecting `logstash-output` plugin clients to provide matching `username` and `password`. + +NOTE: when SSL is disabled, data and credentials will be received in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Logstash Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type |Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string] |No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number] |No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + +* Value type is a {logstash-ref}/configuration-file-structure.html#string[string] ip address +* Default value is `0.0.0.0` (all interfaces) + +Specify which interface to listen on by providing its ip address. +By default, this input listens on all available interfaces. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Password for password-based authentication. +Requires <<{version}-plugins-{type}s-{plugin}-username>>. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + +* Value type is a {logstash-ref}/configuration-file-structure.html#number[number] port +* Default value is 9800 + +Specify which port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* Cannot be combined with configurations that disable SSL. + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting clients. +The certificate _SHOULD_ include identity claims about the ip address or hostname that clients use to establish a connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication, `+ssl_client_authentication => none+`>>. + +One or more PEM-encoded files defining certificate authorities for use in client authentication. +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: +** `none`: do not request client's certificate, or validate certificates that are presented +** `optional`: request client's certificate, and validate it against our trust authorities _if-and-only-if_ it is presented +** `required`: require a valid certificate from the client that is signed by a trusted certificate authority +* Default value is `"none"` + +By default the server doesn't do any client authentication. +This means that connections from clients are _private_ when SSL is enabled, but that this input will allow SSL connections from _any_ client. +If you wish to configure this plugin to reject connections from untrusted hosts, you will need to configure this plugin to authenticate clients, and may also need to configure its <<{version}-plugins-{type}s-{plugin}-config-ssl-trust, source of trust>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +SSL is enabled by default, which requires configuring this plugin to present its <<{version}-plugins-{type}s-{plugin}-config-ssl-identity, identity>>. + +You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. +* Cannot be combined with configurations that disable SSL. + +A path to a PEM-encoded _encrypted_ PKCS8 SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* Cannot be combined with configurations that disable SSL. + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. +* Cannot be combined with configurations that disable SSL. + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Username for password-based authentication. +When this input plugin is configured with a `username`, it also requires a `password`, and any upstream `logstash-output` plugin must also be configured with a matching `username`/`password` pair. + +NOTE: when SSL is disabled, credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/logstash-v1.0.4.asciidoc b/docs/versioned-plugins/inputs/logstash-v1.0.4.asciidoc new file mode 100644 index 000000000..80b6202f1 --- /dev/null +++ b/docs/versioned-plugins/inputs/logstash-v1.0.4.asciidoc @@ -0,0 +1,250 @@ +:integration: logstash +:plugin: logstash +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.4 +:release_date: 2024-12-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Listen for events that are sent by a {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin] in a pipeline that may be in another process or on another host. +The upstream output must have a TCP route to the port (defaults to 9800) on an interface that this plugin is bound to. + +NOTE: Sending events to this input by _any_ means other than `plugins-outputs-logstash` is neither advised nor supported. +We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding. + +[id="{version}-plugins-{type}s-{plugin}-minimum-config"] +===== Minimum Configuration +[cols="3a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +input { + logstash { + ssl_keystore_path + => "/path/to/logstash.p12" + ssl_keystore_password + => "${PASS}" + } +} +---- + +| + +[source,shell] +---- +input { + logstash { + ssl_enabled => false + } +} +---- + +|======================================================================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-config-binding"] +===== Configuration Concepts + +This input plugin needs to be configured to bind to a TCP <<{version}-plugins-{type}s-{plugin}-port>>, and can be constrained to bind to a particular interface by providing the IP to <<{version}-plugins-{type}s-{plugin}-host>>. + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-identity"] +===== Security: SSL Identity + +Unless SSL is disabled, this plugin needs to be configured with identity material: + +* JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) +* PKCS8-formatted Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-trust"] +===== Security: SSL Trust + +When communicating over SSL, this plugin can be configured to either request or require that connecting clients present their own identity claims with <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>>. + +Certificates that are presented by clients are validated by default using the system trust store to ensure that they are currently-valid and trusted, and that the client can prove possession of its associated private key. +You can provide an _alternate_ source of trust with: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) + +NOTE: Client-certificate verification does _not_ verify identity claims on the presented certificate, such as whether the certificate includes a Subject Alt Name matching the IP address from which the client is connecting. + +[id="{version}-plugins-{type}s-{plugin}-config-credentials"] +===== Security: Credentials + +You can also configure this plugin to require a specific username/password be provided by configuring <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>. +Doing so requires connecting `logstash-output` plugin clients to provide matching `username` and `password`. + +NOTE: when SSL is disabled, data and credentials will be received in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Logstash Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type |Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string] |No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number] |No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + +* Value type is a {logstash-ref}/configuration-file-structure.html#string[string] ip address +* Default value is `0.0.0.0` (all interfaces) + +Specify which interface to listen on by providing its ip address. +By default, this input listens on all available interfaces. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Password for password-based authentication. +Requires <<{version}-plugins-{type}s-{plugin}-username>>. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + +* Value type is a {logstash-ref}/configuration-file-structure.html#number[number] port +* Default value is 9800 + +Specify which port to listen on. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* Cannot be combined with configurations that disable SSL. + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting clients. +The certificate _SHOULD_ include identity claims about the ip address or hostname that clients use to establish a connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication, `+ssl_client_authentication => none+`>>. + +One or more PEM-encoded files defining certificate authorities for use in client authentication. +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + +* Value can be any of: +** `none`: do not request client's certificate, or validate certificates that are presented +** `optional`: request client's certificate, and validate it against our trust authorities _if-and-only-if_ it is presented +** `required`: require a valid certificate from the client that is signed by a trusted certificate authority +* Default value is `"none"` + +By default the server doesn't do any client authentication. +This means that connections from clients are _private_ when SSL is enabled, but that this input will allow SSL connections from _any_ client. +If you wish to configure this plugin to reject connections from untrusted hosts, you will need to configure this plugin to authenticate clients, and may also need to configure its <<{version}-plugins-{type}s-{plugin}-config-ssl-trust, source of trust>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +SSL is enabled by default, which requires configuring this plugin to present its <<{version}-plugins-{type}s-{plugin}-config-ssl-identity, identity>>. + +You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. +* Cannot be combined with configurations that disable SSL. + +A path to a PEM-encoded _encrypted_ PKCS8 SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* Cannot be combined with configurations that disable SSL. + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. +* Cannot be combined with configurations that disable SSL. + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +Username for password-based authentication. +When this input plugin is configured with a `username`, it also requires a `password`, and any upstream `logstash-output` plugin must also be configured with a matching `username`/`password` pair. + +NOTE: when SSL is disabled, credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/rabbitmq-index.asciidoc b/docs/versioned-plugins/inputs/rabbitmq-index.asciidoc index bbc0232ec..1c7ff0046 100644 --- a/docs/versioned-plugins/inputs/rabbitmq-index.asciidoc +++ b/docs/versioned-plugins/inputs/rabbitmq-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-09-16 | <> | 2023-06-09 | <> | 2023-05-30 | <> | 2022-10-04 @@ -25,6 +26,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::rabbitmq-v7.4.0.asciidoc[] include::rabbitmq-v7.3.3.asciidoc[] include::rabbitmq-v7.3.2.asciidoc[] include::rabbitmq-v7.3.1.asciidoc[] diff --git a/docs/versioned-plugins/inputs/rabbitmq-v7.4.0.asciidoc b/docs/versioned-plugins/inputs/rabbitmq-v7.4.0.asciidoc new file mode 100644 index 000000000..20363e2ce --- /dev/null +++ b/docs/versioned-plugins/inputs/rabbitmq-v7.4.0.asciidoc @@ -0,0 +1,425 @@ +:integration: rabbitmq +:plugin: rabbitmq +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.4.0 +:release_date: 2024-09-16 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-rabbitmq/blob/v7.4.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Rabbitmq input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Pull events from a http://www.rabbitmq.com/[RabbitMQ] queue. + +The default settings will create an entirely transient queue and listen for all messages by default. +If you need durability or any other advanced settings, please set the appropriate options + +This plugin uses the http://rubymarchhare.info/[March Hare] library +for interacting with the RabbitMQ server. Most configuration options +map directly to standard RabbitMQ and AMQP concepts. The +https://www.rabbitmq.com/amqp-0-9-1-reference.html[AMQP 0-9-1 reference guide] +and other parts of the RabbitMQ documentation are useful for deeper +understanding. + +The properties of messages received will be stored in the +`[@metadata][rabbitmq_properties]` field if the `@metadata_enabled` setting is enabled. +Note that storing metadata may degrade performance. +The following properties may be available (in most cases dependent on whether +they were set by the sender): + +* app-id +* cluster-id +* consumer-tag +* content-encoding +* content-type +* correlation-id +* delivery-mode +* exchange +* expiration +* message-id +* priority +* redeliver +* reply-to +* routing-key +* timestamp +* type +* user-id + +For example, to get the RabbitMQ message's timestamp property +into the Logstash event's `@timestamp` field, use the date +filter to parse the `[@metadata][rabbitmq_properties][timestamp]` +field: +[source,ruby] + filter { + if [@metadata][rabbitmq_properties][timestamp] { + date { + match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"] + } + } + } + +Additionally, any message headers will be saved in the +`[@metadata][rabbitmq_headers]` field. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Rabbitmq Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ack>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-arguments>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-auto_delete>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-automatic_recovery>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-connect_retry_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-durable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exchange>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exchange_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exclusive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-heartbeat>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-passive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-prefetch_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_password>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-subscription_retry_interval_seconds>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-vhost>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-ack"] +===== `ack` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable message acknowledgements. With acknowledgements +messages fetched by Logstash but not yet sent into the +Logstash pipeline will be requeued by the server if Logstash +shuts down. Acknowledgements will however hurt the message +throughput. + +This will only send an ack back every `prefetch_count` messages. +Working in batches provides a performance boost here. + +[id="{version}-plugins-{type}s-{plugin}-arguments"] +===== `arguments` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +Optional queue arguments as an array. + +Relevant RabbitMQ doc guides: + + * https://www.rabbitmq.com/queues.html#optional-arguments[Optional queue arguments] + * https://www.rabbitmq.com/parameters.html#policies[Policies] + * https://www.rabbitmq.com/quorum-queues.html[Quorum Queues] + +[id="{version}-plugins-{type}s-{plugin}-auto_delete"] +===== `auto_delete` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Should the queue be deleted on the broker when the last consumer +disconnects? Set this option to `false` if you want the queue to remain +on the broker, queueing up messages until a consumer comes along to +consume them. + +[id="{version}-plugins-{type}s-{plugin}-automatic_recovery"] +===== `automatic_recovery` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Set this to https://www.rabbitmq.com/connections.html#automatic-recovery[automatically recover] from a broken connection. +You almost certainly don't want to override this! + +[id="{version}-plugins-{type}s-{plugin}-connect_retry_interval"] +===== `connect_retry_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Time in seconds to wait before retrying a connection + +[id="{version}-plugins-{type}s-{plugin}-connection_timeout"] +===== `connection_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default connection timeout in milliseconds. If not specified the timeout is infinite. + +[id="{version}-plugins-{type}s-{plugin}-durable"] +===== `durable` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Is this queue durable? (aka; Should it survive a broker restart?) +If consuming directly from a queue you must set this value to match the existing queue setting, +otherwise the connection will fail due to an inequivalent arg error. + +[id="{version}-plugins-{type}s-{plugin}-exchange"] +===== `exchange` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the exchange to bind the queue to. Specify `exchange_type` +as well to declare the exchange if it does not exist + +[id="{version}-plugins-{type}s-{plugin}-exchange_type"] +===== `exchange_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The type of the exchange to bind to. Specifying this will cause this plugin +to declare the exchange if it does not exist. + +[id="{version}-plugins-{type}s-{plugin}-exclusive"] +===== `exclusive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Is the queue exclusive? Exclusive queues can only be used by the connection +that declared them and will be deleted when it is closed (e.g. due to a Logstash +restart). + +[id="{version}-plugins-{type}s-{plugin}-heartbeat"] +===== `heartbeat` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +https://www.rabbitmq.com/heartbeats.html[Heartbeat timeout] in seconds. +If unspecified then heartbeat timeout of 60 seconds will be used. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Common functionality for the rabbitmq input/output +RabbitMQ server address(es) +host can either be a single host, or a list of hosts +i.e. + host => "localhost" +or + host => ["host01", "host02] + +if multiple hosts are provided on the initial connection and any subsequent +recovery attempts of the hosts is chosen at random and connected to. +Note that only one host connection is active at a time. + +[id="{version}-plugins-{type}s-{plugin}-key"] +===== `key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The routing key to use when binding a queue to the exchange. +This is only relevant for direct or topic exchanges. + +* Routing keys are ignored on fanout exchanges. +* Wildcards are not valid on direct exchanges. + +[id="{version}-plugins-{type}s-{plugin}-metadata_enabled"] +===== `metadata_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Accepted values are: + - `none`: no metadata is added + - `basic`: headers and properties are added + - `extended`: headers, properties, and raw payload are added + - `false`: deprecated alias for `none` + - `true`: deprecated alias for `basic` + * Default value is `none` + +Enable metadata about the RabbitMQ topic to be added to the event's `@metadata` field, for availablity during pipeline processing. In general, most output plugins and codecs do not include `@metadata` fields. This may impact memory usage and performance. + +[id="{version}-plugins-{type}s-{plugin}-metadata_locations"] +===== Metadata mapping +|===== +| category | location | type + +| headers |`[@metadata][rabbitmq_headers]` | key/value map +| properties |`[@metadata][rabbitmq_properties]` | key/value map +| raw payload |`[@metadata][rabbitmq_payload]` | byte sequence +|===== + + +[id="{version}-plugins-{type}s-{plugin}-passive"] +===== `passive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If true the queue will be passively declared, meaning it must +already exist on the server. To have Logstash create the queue +if necessary leave this option as false. If actively declaring +a queue that already exists, the queue options for this plugin +(durable etc) must match those of the existing queue. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `"guest"` + +RabbitMQ password + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5672` + +RabbitMQ port to connect on + +[id="{version}-plugins-{type}s-{plugin}-prefetch_count"] +===== `prefetch_count` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `256` + +Prefetch count. If acknowledgements are enabled with the `ack` +option, specifies the number of outstanding unacknowledged +messages allowed. + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +The properties to extract from each message and store in a +@metadata field. + +Technically the exchange, redeliver, and routing-key +properties belong to the envelope and not the message but we +ignore that distinction here. However, we extract the +headers separately via get_headers even though the header +table technically is a message property. + +Freezing all strings so that code modifying the event's +@metadata field can't touch them. + +If updating this list, remember to update the documentation +above too. +The default codec for this plugin is JSON. You can override this to suit your particular needs however. +The name of the queue Logstash will consume events from. If +left empty, a transient queue with an randomly chosen name +will be created. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable or disable SSL. +Note that by default remote certificate verification is off. +Specify ssl_certificate_path and ssl_certificate_password if you need +certificate verification + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_password"] +===== `ssl_certificate_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Password for the encrypted PKCS12 (.p12) certificate file specified in ssl_certificate_path + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_path"] +===== `ssl_certificate_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to an SSL certificate in PKCS12 (.p12) format used for verifying the remote host + +[id="{version}-plugins-{type}s-{plugin}-ssl_version"] +===== `ssl_version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"TLSv1.2"` + +Version of the SSL protocol to use. + +[id="{version}-plugins-{type}s-{plugin}-subscription_retry_interval_seconds"] +===== `subscription_retry_interval_seconds` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Amount of time in seconds to wait after a failed subscription request +before retrying. Subscribes can fail if the server goes away and then comes back. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"guest"` + +RabbitMQ username + +[id="{version}-plugins-{type}s-{plugin}-vhost"] +===== `vhost` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/"` + +The vhost (virtual host) to use. If you don't know what this +is, leave the default. With the exception of the default +vhost ("/"), names of vhosts should not begin with a forward +slash. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/redis-index.asciidoc b/docs/versioned-plugins/inputs/redis-index.asciidoc index 06d592783..b4f04e188 100644 --- a/docs/versioned-plugins/inputs/redis-index.asciidoc +++ b/docs/versioned-plugins/inputs/redis-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-08-01 | <> | 2021-05-04 | <> | 2021-04-07 | <> | 2020-12-22 @@ -20,6 +21,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::redis-v3.7.1.asciidoc[] include::redis-v3.7.0.asciidoc[] include::redis-v3.6.1.asciidoc[] include::redis-v3.6.0.asciidoc[] diff --git a/docs/versioned-plugins/inputs/redis-v3.7.1.asciidoc b/docs/versioned-plugins/inputs/redis-v3.7.1.asciidoc new file mode 100644 index 000000000..3485c5004 --- /dev/null +++ b/docs/versioned-plugins/inputs/redis-v3.7.1.asciidoc @@ -0,0 +1,175 @@ +:plugin: redis +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.7.1 +:release_date: 2024-08-01 +:changelog_url: https://github.com/logstash-plugins/logstash-input-redis/blob/v3.7.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Redis input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This input will read events from a Redis instance; it supports both Redis channels and lists. +The list command (BLPOP) used by Logstash is supported in Redis v1.3.1+, and +the channel commands used by Logstash are found in Redis v1.3.8+. +While you may be able to make these Redis versions work, the best performance +and stability will be found in more recent stable versions. Versions 2.6.0+ +are recommended. + +For more information about Redis, see + +`batch_count` note: If you use the `batch_count` setting, you *must* use a Redis version 2.6.0 or +newer. Anything older does not support the operations used by batching. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Redis Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-batch_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-command_map>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["list", "channel", "pattern_channel"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-db>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-key>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-batch_count"] +===== `batch_count` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `125` + +The number of events to return from Redis using EVAL. + +[id="{version}-plugins-{type}s-{plugin}-command_map"] +===== `command_map` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + * key is the default command name, value is the renamed command. + +Configure renamed redis commands in the form of "oldname" => "newname". +Redis allows for the renaming or disabling of commands in its protocol, see: https://redis.io/topics/security + + +[id="{version}-plugins-{type}s-{plugin}-data_type"] +===== `data_type` + + * This is a required setting. + * Value can be any of: `list`, `channel`, `pattern_channel` + * There is no default value for this setting. + +Specify either list or channel. If `data_type` is `list`, then we will BLPOP the +key. If `data_type` is `channel`, then we will SUBSCRIBE to the key. +If `data_type` is `pattern_channel`, then we will PSUBSCRIBE to the key. + +[id="{version}-plugins-{type}s-{plugin}-db"] +===== `db` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The Redis database number. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"127.0.0.1"` + +The hostname of your Redis server. + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * Path will override Host configuration if both specified. + +The unix socket path of your Redis server. + +[id="{version}-plugins-{type}s-{plugin}-key"] +===== `key` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of a Redis list or channel. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate with. There is no authentication by default. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `6379` + +The port to connect on. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL support. + + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Number of instances of the input to start, each on its own thread. +Increase from one to improve concurrency in consuming messages from Redis. + +NOTE: Increasing the number of threads when consuming from a channel will result in duplicate messages since a `SUBSCRIBE` delivers each message to all subscribers. + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Initial connection timeout in seconds. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/s3-index.asciidoc b/docs/versioned-plugins/inputs/s3-index.asciidoc index 77e9fef68..45c53d53e 100644 --- a/docs/versioned-plugins/inputs/s3-index.asciidoc +++ b/docs/versioned-plugins/inputs/s3-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -40,6 +44,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::s3-v7.2.1.asciidoc[] +include::s3-v7.2.0.asciidoc[] +include::s3-v7.1.8.asciidoc[] +include::s3-v7.1.7.asciidoc[] include::s3-v7.1.6.asciidoc[] include::s3-v7.1.5.asciidoc[] include::s3-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/inputs/s3-v7.1.7.asciidoc b/docs/versioned-plugins/inputs/s3-v7.1.7.asciidoc new file mode 100644 index 000000000..225d24af3 --- /dev/null +++ b/docs/versioned-plugins/inputs/s3-v7.1.7.asciidoc @@ -0,0 +1,357 @@ +:integration: aws +:plugin: s3 +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Stream events from files from a S3 bucket. + +IMPORTANT: The S3 input plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +Each line from each file generates an event. +Files ending in `.gz` are handled as gzip'ed files. + +Files that are archived to AWS Glacier will be skipped. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +This plugin adds cloudfront metadata to event. +When ECS compatibility is disabled, the value is stored in the root level. +When ECS is enabled, the value is stored in the `@metadata` where it can be used by other plugins in your pipeline. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_add_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_dir>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-delete>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-gzip_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_object_properties>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sincedb_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-watch_for_new_files>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to s3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + s3 { + access_key_id => "1234" + secret_access_key => "secret" + bucket => "logstash-test" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-backup_add_prefix"] +===== `backup_add_prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Append a prefix to the key (full path including file name in s3) after processing. +If backing up to another (or the same) bucket, this effectively lets you +choose a new 'folder' to place the files in + +[id="{version}-plugins-{type}s-{plugin}-backup_to_bucket"] +===== `backup_to_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Name of a S3 bucket to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-backup_to_dir"] +===== `backup_to_dir` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Path of a local directory to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the S3 bucket. + +[id="{version}-plugins-{type}s-{plugin}-delete"] +===== `delete` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether to delete processed files from the original bucket. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names + ** `v1`,`v8`: uses metadata fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-exclude_pattern"] +===== `exclude_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Ruby style regexp of keys to exclude from the bucket. + +Note that files matching the pattern are skipped _after_ they have been listed. +Consider using <<{version}-plugins-{type}s-{plugin}-prefix>> instead where possible. + +Example: + +[source,ruby] +----- +"exclude_pattern" => "\/2020\/04\/" +----- + +This pattern excludes all logs containing "/2020/04/" in the path. + + +[id="{version}-plugins-{type}s-{plugin}-gzip_pattern"] +===== `gzip_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\.gz(ip)?$"` + +Regular expression used to determine whether an input file is in gzip format. + +[id="{version}-plugins-{type}s-{plugin}-include_object_properties"] +===== `include_object_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether or not to include the S3 object's properties (last_modified, content_type, metadata) into each Event at +`[@metadata][s3]`. Regardless of this setting, `[@metadata][s3][key]` will always be present. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Interval to wait between to check the file list again after a run is finished. +Value is in seconds. + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +If specified, the prefix of filenames in the bucket must match (not a regexp) + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-sincedb_path"] +===== `sincedb_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Where to write the since database (keeps track of the date +the last handled file was added to S3). The default will write +sincedb files to in the directory '{path.data}/plugins/inputs/s3/' + +If specified, this setting must be a filename path and not just a directory. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before processing them. + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-watch_for_new_files"] +===== `watch_for_new_files` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether or not to watch for new files. +Disabling this option causes the input to close itself after processing the files from a single listing. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/s3-v7.1.8.asciidoc b/docs/versioned-plugins/inputs/s3-v7.1.8.asciidoc new file mode 100644 index 000000000..092fd2384 --- /dev/null +++ b/docs/versioned-plugins/inputs/s3-v7.1.8.asciidoc @@ -0,0 +1,357 @@ +:integration: aws +:plugin: s3 +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Stream events from files from a S3 bucket. + +IMPORTANT: The S3 input plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +Each line from each file generates an event. +Files ending in `.gz` are handled as gzip'ed files. + +Files that are archived to AWS Glacier will be skipped. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +This plugin adds cloudfront metadata to event. +When ECS compatibility is disabled, the value is stored in the root level. +When ECS is enabled, the value is stored in the `@metadata` where it can be used by other plugins in your pipeline. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_add_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_dir>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-delete>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-gzip_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_object_properties>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sincedb_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-watch_for_new_files>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to s3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + s3 { + access_key_id => "1234" + secret_access_key => "secret" + bucket => "logstash-test" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-backup_add_prefix"] +===== `backup_add_prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Append a prefix to the key (full path including file name in s3) after processing. +If backing up to another (or the same) bucket, this effectively lets you +choose a new 'folder' to place the files in + +[id="{version}-plugins-{type}s-{plugin}-backup_to_bucket"] +===== `backup_to_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Name of a S3 bucket to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-backup_to_dir"] +===== `backup_to_dir` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Path of a local directory to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the S3 bucket. + +[id="{version}-plugins-{type}s-{plugin}-delete"] +===== `delete` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether to delete processed files from the original bucket. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names + ** `v1`,`v8`: uses metadata fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-exclude_pattern"] +===== `exclude_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Ruby style regexp of keys to exclude from the bucket. + +Note that files matching the pattern are skipped _after_ they have been listed. +Consider using <<{version}-plugins-{type}s-{plugin}-prefix>> instead where possible. + +Example: + +[source,ruby] +----- +"exclude_pattern" => "\/2020\/04\/" +----- + +This pattern excludes all logs containing "/2020/04/" in the path. + + +[id="{version}-plugins-{type}s-{plugin}-gzip_pattern"] +===== `gzip_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\.gz(ip)?$"` + +Regular expression used to determine whether an input file is in gzip format. + +[id="{version}-plugins-{type}s-{plugin}-include_object_properties"] +===== `include_object_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether or not to include the S3 object's properties (last_modified, content_type, metadata) into each Event at +`[@metadata][s3]`. Regardless of this setting, `[@metadata][s3][key]` will always be present. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Interval to wait between to check the file list again after a run is finished. +Value is in seconds. + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +If specified, the prefix of filenames in the bucket must match (not a regexp) + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-sincedb_path"] +===== `sincedb_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Where to write the since database (keeps track of the date +the last handled file was added to S3). The default will write +sincedb files to in the directory '{path.data}/plugins/inputs/s3/' + +If specified, this setting must be a filename path and not just a directory. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before processing them. + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-watch_for_new_files"] +===== `watch_for_new_files` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether or not to watch for new files. +Disabling this option causes the input to close itself after processing the files from a single listing. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/s3-v7.2.0.asciidoc b/docs/versioned-plugins/inputs/s3-v7.2.0.asciidoc new file mode 100644 index 000000000..f339524a0 --- /dev/null +++ b/docs/versioned-plugins/inputs/s3-v7.2.0.asciidoc @@ -0,0 +1,369 @@ +:integration: aws +:plugin: s3 +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Stream events from files from a S3 bucket. + +IMPORTANT: The S3 input plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +Each line from each file generates an event. +Files ending in `.gz` are handled as gzip'ed files. + +Files that are archived to AWS Glacier will be skipped. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +This plugin adds cloudfront metadata to event. +When ECS compatibility is disabled, the value is stored in the root level. +When ECS is enabled, the value is stored in the `@metadata` where it can be used by other plugins in your pipeline. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_add_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_dir>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-delete>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-gzip_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_object_properties>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sincedb_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-watch_for_new_files>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to s3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + s3 { + access_key_id => "1234" + secret_access_key => "secret" + bucket => "logstash-test" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-backup_add_prefix"] +===== `backup_add_prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Append a prefix to the key (full path including file name in s3) after processing. +If backing up to another (or the same) bucket, this effectively lets you +choose a new 'folder' to place the files in + +[id="{version}-plugins-{type}s-{plugin}-backup_to_bucket"] +===== `backup_to_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Name of a S3 bucket to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-backup_to_dir"] +===== `backup_to_dir` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Path of a local directory to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the S3 bucket. + +[id="{version}-plugins-{type}s-{plugin}-delete"] +===== `delete` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether to delete processed files from the original bucket. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names + ** `v1`,`v8`: uses metadata fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-exclude_pattern"] +===== `exclude_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Ruby style regexp of keys to exclude from the bucket. + +Note that files matching the pattern are skipped _after_ they have been listed. +Consider using <<{version}-plugins-{type}s-{plugin}-prefix>> instead where possible. + +Example: + +[source,ruby] +----- +"exclude_pattern" => "\/2020\/04\/" +----- + +This pattern excludes all logs containing "/2020/04/" in the path. + + +[id="{version}-plugins-{type}s-{plugin}-gzip_pattern"] +===== `gzip_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\.gz(ip)?$"` + +Regular expression used to determine whether an input file is in gzip format. + +[id="{version}-plugins-{type}s-{plugin}-include_object_properties"] +===== `include_object_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether or not to include the S3 object's properties (last_modified, content_type, metadata) into each Event at +`[@metadata][s3]`. Regardless of this setting, `[@metadata][s3][key]` will always be present. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Interval to wait between to check the file list again after a run is finished. +Value is in seconds. + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +If specified, the prefix of filenames in the bucket must match (not a regexp) + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-sincedb_path"] +===== `sincedb_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Where to write the since database (keeps track of the date +the last handled file was added to S3). The default will write +sincedb files to in the directory '{path.data}/plugins/inputs/s3/' + +If specified, this setting must be a filename path and not just a directory. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before processing them. + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-watch_for_new_files"] +===== `watch_for_new_files` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether or not to watch for new files. +Disabling this option causes the input to close itself after processing the files from a single listing. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/s3-v7.2.1.asciidoc b/docs/versioned-plugins/inputs/s3-v7.2.1.asciidoc new file mode 100644 index 000000000..d63cf58eb --- /dev/null +++ b/docs/versioned-plugins/inputs/s3-v7.2.1.asciidoc @@ -0,0 +1,369 @@ +:integration: aws +:plugin: s3 +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Stream events from files from a S3 bucket. + +IMPORTANT: The S3 input plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +Each line from each file generates an event. +Files ending in `.gz` are handled as gzip'ed files. + +Files that are archived to AWS Glacier will be skipped. + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +This plugin adds cloudfront metadata to event. +When ECS compatibility is disabled, the value is stored in the root level. +When ECS is enabled, the value is stored in the `@metadata` where it can be used by other plugins in your pipeline. + +Here’s how ECS compatibility mode affects output. +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_add_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-backup_to_dir>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-delete>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-exclude_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-gzip_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-include_object_properties>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sincedb_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-watch_for_new_files>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to s3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + s3 { + access_key_id => "1234" + secret_access_key => "secret" + bucket => "logstash-test" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-backup_add_prefix"] +===== `backup_add_prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Append a prefix to the key (full path including file name in s3) after processing. +If backing up to another (or the same) bucket, this effectively lets you +choose a new 'folder' to place the files in + +[id="{version}-plugins-{type}s-{plugin}-backup_to_bucket"] +===== `backup_to_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Name of a S3 bucket to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-backup_to_dir"] +===== `backup_to_dir` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Path of a local directory to backup processed files to. + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the S3 bucket. + +[id="{version}-plugins-{type}s-{plugin}-delete"] +===== `delete` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether to delete processed files from the original bucket. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names + ** `v1`,`v8`: uses metadata fields that are compatible with Elastic Common Schema + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <<{version}-plugins-{type}s-{plugin}-ecs_metadata>> for detailed information. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-exclude_pattern"] +===== `exclude_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Ruby style regexp of keys to exclude from the bucket. + +Note that files matching the pattern are skipped _after_ they have been listed. +Consider using <<{version}-plugins-{type}s-{plugin}-prefix>> instead where possible. + +Example: + +[source,ruby] +----- +"exclude_pattern" => "\/2020\/04\/" +----- + +This pattern excludes all logs containing "/2020/04/" in the path. + + +[id="{version}-plugins-{type}s-{plugin}-gzip_pattern"] +===== `gzip_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"\.gz(ip)?$"` + +Regular expression used to determine whether an input file is in gzip format. + +[id="{version}-plugins-{type}s-{plugin}-include_object_properties"] +===== `include_object_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether or not to include the S3 object's properties (last_modified, content_type, metadata) into each Event at +`[@metadata][s3]`. Regardless of this setting, `[@metadata][s3][key]` will always be present. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Interval to wait between to check the file list again after a run is finished. +Value is in seconds. + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +If specified, the prefix of filenames in the bucket must match (not a regexp) + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-sincedb_path"] +===== `sincedb_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Where to write the since database (keeps track of the date +the last handled file was added to S3). The default will write +sincedb files to in the directory '{path.data}/plugins/inputs/s3/' + +If specified, this setting must be a filename path and not just a directory. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before processing them. + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-watch_for_new_files"] +===== `watch_for_new_files` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether or not to watch for new files. +Disabling this option causes the input to close itself after processing the files from a single listing. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/salesforce-index.asciidoc b/docs/versioned-plugins/inputs/salesforce-index.asciidoc index 52bce1331..85385ce3f 100644 --- a/docs/versioned-plugins/inputs/salesforce-index.asciidoc +++ b/docs/versioned-plugins/inputs/salesforce-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-05-14 | <> | 2023-05-30 | <> | 2021-12-07 | <> | 2021-11-11 @@ -15,6 +16,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::salesforce-v3.3.0.asciidoc[] include::salesforce-v3.2.1.asciidoc[] include::salesforce-v3.2.0.asciidoc[] include::salesforce-v3.1.0.asciidoc[] diff --git a/docs/versioned-plugins/inputs/salesforce-v3.3.0.asciidoc b/docs/versioned-plugins/inputs/salesforce-v3.3.0.asciidoc new file mode 100644 index 000000000..594358ced --- /dev/null +++ b/docs/versioned-plugins/inputs/salesforce-v3.3.0.asciidoc @@ -0,0 +1,348 @@ +:plugin: salesforce +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.3.0 +:release_date: 2025-05-14 +:changelog_url: https://github.com/logstash-plugins/logstash-input-salesforce/blob/v3.3.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Salesforce input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This Logstash input plugin allows you to query Salesforce using SOQL and puts the results +into Logstash, one row per event. You can configure it to pull entire sObjects or only +specific fields. + +NOTE: By default, this input plugin will stop after all the results of the query are processed and will +need to be re-run to fetch new results. It does not utilize the streaming API. However, by setting the `interval` +configuration option you can configure the plugin to automatically run at a set frequency. + +In order to use this plugin, you will need to create a new Salesforce Connected App with OAuth enabled. +More details can be found here: +https://help.salesforce.com/apex/HTViewHelpDoc?id=connected_app_create.htm + +You will also need a username, password, and security token for your Salesforce instance. +More details for generating a token can be found here: +https://help.salesforce.com/apex/HTViewHelpDoc?id=user_security_token.htm + +In addition to specifying an sObject, you can also supply a list of API fields +that will be used in the SOQL query. + +==== HTTP proxy + +If your infrastructure uses an HTTP proxy, you can set the `SALESFORCE_PROXY_URI` environment variable with the desired URI value (e.g `export SALESFORCE_PROXY_URI="http://proxy.example.com:123"`). + +==== Example +This example prints all the Salesforce Opportunities to standard out + +[source,ruby] +---------------------------------- +input { + salesforce { + client_id => 'OAUTH CLIENT ID FROM YOUR SFDC APP' + client_secret => 'OAUTH CLIENT SECRET FROM YOUR SFDC APP' + username => 'email@example.com' + password => 'super-secret' + security_token => 'SECURITY TOKEN FOR THIS USER' + sfdc_object_name => 'Opportunity' + } +} + +output { + stdout { + codec => rubydebug + } +} +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Salesforce Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-changed_data_filter>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-client_secret>> |{logstash-ref}/configuration-file-structure.html#password[password]|Yes +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|Yes +| <<{version}-plugins-{type}s-{plugin}-security_token>> |{logstash-ref}/configuration-file-structure.html#password[password]|Yes +| <<{version}-plugins-{type}s-{plugin}-sfdc_fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sfdc_filters>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sfdc_instance_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sfdc_object_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-to_underscores>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_field_value_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_test_sandbox>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_tooling_api>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_version"] +===== `api_version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +By default, this uses the default Restforce API version. +To override this, set this to something like "32.0" for example. + +[id="{version}-plugins-{type}s-{plugin}-changed_data_filter"] +===== `changed_data_filter` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The filter to add to the Salesforce query when a previous tracking field value +was read from the <<{version}-plugins-{type}s-{plugin}-tracking_field_value_file>>. +The string can (and should) contain a placeholder `%+{last_tracking_field_value}+` that +will be substituted with the actual value read from the <<{version}-plugins-{type}s-{plugin}-tracking_field_value_file>>. + +This clause is combined with any <<{version}-plugins-{type}s-{plugin}-sfdc_filters>> +clause that is configured using the `AND` operator. + +The value should be properly quoted according to the SOQL rules for the field +type. + +**Examples:** + +[source,ruby] +"changed_data_filter" => "Number > '%{last_tracking_field_value}'" + +[source,ruby] +"changed_data_filter" => "LastModifiedDate >= %{last_tracking_field_value}" + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Consumer Key for authentication. You must set up a new Salesforce +connected app with OAuth enabled to use this plugin. More information +can be found here: +https://help.salesforce.com/apex/HTViewHelpDoc?id=connected_app_create.htm. + +[id="{version}-plugins-{type}s-{plugin}-client_secret"] +===== `client_secret` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Consumer secret from your OAuth enabled connected app. + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The interval in seconds between each run of the plugin. + +If specified, the plugin only terminates when it receives the stop +signal from Logstash, e.g. when you press Ctrl-C when running interactively, +or when the process receives a TERM signal. It will query and publish +events for all results, then sleep until `interval` seconds from the start +of the previous run of the plugin have passed. If the plugin ran for longer +than `interval` seconds, it will run again immediately. + +If this property is not specified or is set to -1, the plugin will run once and then exit. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password used to log in to Salesforce. + +[id="{version}-plugins-{type}s-{plugin}-security_token"] +===== `security_token` + +* This is a required setting. +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The security token for this account. For more information about +generating a security token, see: +https://help.salesforce.com/apex/HTViewHelpDoc?id=user_security_token.htm. + +[id="{version}-plugins-{type}s-{plugin}-sfdc_fields"] +===== `sfdc_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +These are the field names to return in the Salesforce query +If this is empty, all fields are returned. + +[id="{version}-plugins-{type}s-{plugin}-sfdc_filters"] +===== `sfdc_filters` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +These options will be added to the `WHERE` clause in the +SOQL statement. Additional fields can be filtered on by +adding `field1 = value1 AND field2 = value2 AND...`. + +[id="{version}-plugins-{type}s-{plugin}-sfdc_instance_url"] +===== `sfdc_instance_url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The url of a Salesforce instance. Provide the url if you want to connect +to your Salesforce instance instead of +`login.salesforce.com` or `test.salesforce.com` at login. + +Use either this or the `use_test_sandbox` configuration option +but not both to configure the url to which the plugin connects to. + +[id="{version}-plugins-{type}s-{plugin}-sfdc_object_name"] +===== `sfdc_object_name` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the Salesforce object you are creating or updating. + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `60` + +The timeout to apply to REST API calls to Salesforce, in seconds. If +a connection to Salesforce cannot be made in this time, an error occurs. +If it takes longer than the timeout for a block of data (e.g. query results) to be +read, an error occurs. + +[id="{version}-plugins-{type}s-{plugin}-to_underscores"] +===== `to_underscores` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting this to true will convert Salesforce's `++NamedFields__c++` to `++named_fields__c++`. + +[id="{version}-plugins-{type}s-{plugin}-tracking_field"] +===== `tracking_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The field to track for incremental data loads. This field will +be used in an `ORDER BY ... ASC` clause that is added to the Salesforce query. +This field _should_ also be used in the <<{version}-plugins-{type}s-{plugin}-changed_data_filter>> clause +to actually achieve incremental loading of data. + +The last value (which is the highest value if the query sorts by this field ascending) +value for this field will be saved to the file at the path configured by +<<{version}-plugins-{type}s-{plugin}-tracking_field_value_file>>, if specified. + +This field should ideally be strictly ascending for new records. An +autonumber field is ideal for this. + +The standard `LastModifiedDate` field can be used, but since it is not _strictly_ +ascending (multiple records can have the same `LastModifiedDate`, the +<<{version}-plugins-{type}s-{plugin}-changed_data_filter>> should account for this by using the `>=` +operator, and duplicates should be expected. + +Note that Salesforce does not guarantee that the standard `Id` field has ascending +values for new records (https://developer.salesforce.com/docs/atlas.en-us.apexcode.meta/apexcode/apex_testing_best_practices.htm). +Therefore, using `Id` as tracking field risks missing records and is not recommended. + +If this field is not already included in the <<{version}-plugins-{type}s-{plugin}-sfdc_fields>>, +it is added. + +[id="{version}-plugins-{type}s-{plugin}-tracking_field_value_file"] +===== `tracking_field_value_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The full path to the file from which the latest tracking field value from the previous +plugin invocation will be read, and to which the new latest tracking field value will be +written after the current plugin invocation. + +This keeps persistent track of the last seen value of the tracking field used for incremental +loading of data. + +The file should be readable and writable by the Logstash process. + +If the file exists and a <<{version}-plugins-{type}s-{plugin}-changed_data_filter>> is configured, +a changed data filter clause is added to the query (and combined with any <<{version}-plugins-{type}s-{plugin}-sfdc_filters>> +clause that is configured using the `AND` operator). + +If the result set is not empty, the value for `tracking_field` from the last row is +written to the file. + +[id="{version}-plugins-{type}s-{plugin}-use_test_sandbox"] +===== `use_test_sandbox` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Set this to true to connect to a sandbox sfdc instance +logging in through test.salesforce.com. + +Use either this or the `sfdc_instance_url` configuration option +but not both to configure the url to which the plugin connects to. + +[id="{version}-plugins-{type}s-{plugin}-use_tooling_api"] +===== `use_tooling_api` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Set this to true to connect to the sfdc tooling api instead of the regular +sfdc rest api. See +https://developer.salesforce.com/docs/atlas.en-us.api_tooling.meta/api_tooling +for details about the sfdc tooling api. Use cases for the sfdc tooling api +include reading apex unit test results, flow coverage results (e.g. coverage +of elements of sfdc flows) and security health check risks. + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A valid Salesforce username, usually your email address. +Used for authentication and will be the user all objects +are created or modified by. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-index.asciidoc b/docs/versioned-plugins/inputs/snmp-index.asciidoc index 3abcd49de..837e635ac 100644 --- a/docs/versioned-plugins/inputs/snmp-index.asciidoc +++ b/docs/versioned-plugins/inputs/snmp-index.asciidoc @@ -5,6 +5,15 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-08-18 +| <> | 2025-07-17 +| <> | 2025-01-23 +| <> | 2025-01-06 +| <> | 2024-07-08 +| <> | 2024-06-11 +| <> | 2024-05-20 +| <> | 2024-05-17 +| <> | 2024-05-03 | <> | 2023-08-07 | <> | 2023-06-30 | <> | 2021-12-23 @@ -23,6 +32,15 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2018-10-24 |======================================================================= +include::snmp-v4.1.0.asciidoc[] +include::snmp-v4.0.7.asciidoc[] +include::snmp-v4.0.6.asciidoc[] +include::snmp-v4.0.5.asciidoc[] +include::snmp-v4.0.4.asciidoc[] +include::snmp-v4.0.3.asciidoc[] +include::snmp-v4.0.2.asciidoc[] +include::snmp-v4.0.1.asciidoc[] +include::snmp-v4.0.0.asciidoc[] include::snmp-v1.3.3.asciidoc[] include::snmp-v1.3.2.asciidoc[] include::snmp-v1.3.1.asciidoc[] diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.0.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.0.asciidoc new file mode 100644 index 000000000..e3db77b77 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.0.asciidoc @@ -0,0 +1,524 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.0 +:release_date: 2024-05-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +// Uncomment after generated the initial versions of the target file +// include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +For migration information and guidelines, check out the migration guide. +**** + +// TODO: Add link to migration topic in the integration index. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.1.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.1.asciidoc new file mode 100644 index 000000000..d4dab5581 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.1.asciidoc @@ -0,0 +1,539 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.1 +:release_date: 2024-05-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of behavioral and mapping differences between current stand-alone plugins and the new versions included in the `integration-snmp`. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +// ToDo: link:{logstash-ref}/plugins-input-snmp.html[Importing MIBs] + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.2.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.2.asciidoc new file mode 100644 index 000000000..81bb892b4 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.2.asciidoc @@ -0,0 +1,539 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.2 +:release_date: 2024-05-20 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of behavioral and mapping differences between current stand-alone plugins and the new versions included in the `integration-snmp`. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +// ToDo: link:{logstash-ref}/plugins-input-snmp.html[Importing MIBs] + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.3.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.3.asciidoc new file mode 100644 index 000000000..fcc79b8f1 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.3.asciidoc @@ -0,0 +1,535 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.3 +:release_date: 2024-06-11 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of behavioral and mapping differences between current stand-alone plugins and the new versions included in the `integration-snmp`. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.4.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.4.asciidoc new file mode 100644 index 000000000..b881b761d --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.4.asciidoc @@ -0,0 +1,524 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.4 +:release_date: 2024-07-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.5.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.5.asciidoc new file mode 100644 index 000000000..ca599de62 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.5.asciidoc @@ -0,0 +1,524 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.5 +:release_date: 2025-01-06 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.6.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.6.asciidoc new file mode 100644 index 000000000..a98bbba97 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.6.asciidoc @@ -0,0 +1,524 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.6 +:release_date: 2025-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.0.7.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.0.7.asciidoc new file mode 100644 index 000000000..6ec6dabde --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.0.7.asciidoc @@ -0,0 +1,524 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.7 +:release_date: 2025-07-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmp-v4.1.0.asciidoc b/docs/versioned-plugins/inputs/snmp-v4.1.0.asciidoc new file mode 100644 index 000000000..8498d0419 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmp-v4.1.0.asciidoc @@ -0,0 +1,524 @@ +:integration: snmp +:plugin: snmp +:type: input +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.0 +:release_date: 2025-08-18 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmp` +**** +The `logstash-input-snmp` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + + +==== Description + +The SNMP input polls network devices using Simple Network Management Protocol (SNMP) +to gather information related to the current state of the devices operation. + +The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols. + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +Metadata fields follow a specific naming convention when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-get>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-local_engine_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-poll_hosts_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-walk>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tables>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-get"] +===== `get` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `get` option to query for scalar values for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options. + +Each host definition is a hash and must define the `host` key and value. +`host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"` + +Each host definition can optionally include the following keys and values: + +* `community` the community string, default is `public`. +* `version` `1`, `2c` or `3`, default is `2c`. +* `retries` is the number of retries in case of failure, default is `2`. +* `timeout` is the timeout in milliseconds with a default value of `1000`. + +*Specifying all hosts options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + } +} +----- + +*Specifying multiple hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}, {host => "udp:192.168.0.1/161" community => "private"}] + } +} +----- + +*Specifying IPv6 hosts* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:[::1]/161" community => "public"}, {host => "udp:[2001:db8::2:1]/161" community => "private"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `30` + +The `interval` option specifies the polling interval in seconds. +If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs. + +[id="{version}-plugins-{type}s-{plugin}-local_engine_id"] +===== `local_engine_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The SNMPv3 local engine ID. Its length must be greater or equal than 5 and less or equal than 32. +If not provided, a default ID is generated based on the local IP address and additional four random bytes. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files, or a +file path to a single MIB file. + +This plugin includes the IETF MIBs. +If you require other MIBs, you need to import them. See <<{version}-plugins-{type}s-{plugin}-import-mibs>>. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: + +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* This setting can be used only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-poll_hosts_timeout"] +===== `poll_hosts_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting + +Specifies the maximum amount of time in milliseconds the polling client will wait for all <<{version}-plugins-{type}s-{plugin}-hosts>> responses. +If all responses are not received before the timeout elapses, the client will fail and some hosts might not get polled during the current cycle. + +By default, it uses the highest value between `1 hour`, the maximum <<{version}-plugins-{type}s-{plugin}-hosts>> configured `timeout`, +and the <<{version}-plugins-{type}s-{plugin}-interval>> value. + +[id="{version}-plugins-{type}s-{plugin}-walk"] +===== `walk` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + +Use the `walk` option to retrieve the subtree of information for the given OID(s). +One or more OID(s) are specified as an array of strings of OID(s). + +Queries the subtree of information starting at the given OID(s). + +Example +[source,ruby] +----- + snmp { + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-tables"] +===== `tables` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting + * Results are returned under a field using the table name + +The `tables` option is used to query for tabular values for the given column OID(s). + +Each table definition is a hash and must define the name key and value and the columns to return. + +*Specifying a single table* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" community => "public" version => "2c" retries => 2 timeout => 1000}] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]} ] + } +} +----- + +*Specifying multiple tables* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + tables => [ {"name" => "interfaces" "columns" => ["1.3.6.1.2.1.2.2.1.1", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.5"]}, {"name" => "ltmPoolStatTable" "columns" => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1", "1.3.6.1.4.1.3375.2.2.5.2.3.1.6"]} ] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is the number of CPU cores + +The number of threads to use for executing the hosts SNMP requests. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +==== SNMPv3 Authentication Options + +A **single user** can be configured and will be used for all defined SNMPv3 hosts. +Multiple snmp input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +[id="{version}-plugins-{type}s-{plugin}-examples"] +==== Configuration examples + +*Specifying SNMPv3 settings* + +[source,ruby] +----- +input { + snmp { + hosts => [{host => "udp:127.0.0.1/161" version => "3"}] + get => ["1.3.6.1.2.1.1.1.0"] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} + +----- + +*Using both `get` and `walk` in the same poll cycle for each host(s)* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"] + walk => ["1.3.6.1.2.1.1"] + hosts => [{host => "udp:127.0.0.1/161" community => "public"}] + } +} +----- + +*Specifying all global options* + +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + mib_paths => ["path/to/converted/mibfile.dic"] + oid_root_skip => 0 + interval => 30 + } +} +----- + +==== Polled host information + +All the polled host information is stored in the event `@metadata`: + +* `[@metadata][host_protocol]` : `udp` or `tcp` +* `[@metadata][host_address]` : host address for example `127.0.0.1` +* `[@metadata][host_port]` : host port (for example `161`) +* `[@metadata][host_community]` : community string for example `public` + + +By default, a `host` field is added to the event with the `[@metadata][host_address]` value. + +[source,ruby] +----- +config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" } +----- + +You can customize the format and content of the `host` field by specifying an alternate `add_field`. + +Example +[source,ruby] +----- +input { + snmp { + get => ["1.3.6.1.2.1.1.1.0"] + hosts => [{host => "udp:127.0.0.1/161"}] + + add_field => {host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}"} + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-index.asciidoc b/docs/versioned-plugins/inputs/snmptrap-index.asciidoc index 0df217bab..46bda355a 100644 --- a/docs/versioned-plugins/inputs/snmptrap-index.asciidoc +++ b/docs/versioned-plugins/inputs/snmptrap-index.asciidoc @@ -5,6 +5,15 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-08-18 +| <> | 2025-07-17 +| <> | 2025-01-23 +| <> | 2025-01-06 +| <> | 2024-07-08 +| <> | 2024-06-11 +| <> | 2024-05-20 +| <> | 2024-05-17 +| <> | 2024-05-03 | <> | 2021-11-19 | <> | 2018-04-06 | <> | 2017-11-07 @@ -12,6 +21,15 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::snmptrap-v4.1.0.asciidoc[] +include::snmptrap-v4.0.7.asciidoc[] +include::snmptrap-v4.0.6.asciidoc[] +include::snmptrap-v4.0.5.asciidoc[] +include::snmptrap-v4.0.4.asciidoc[] +include::snmptrap-v4.0.3.asciidoc[] +include::snmptrap-v4.0.2.asciidoc[] +include::snmptrap-v4.0.1.asciidoc[] +include::snmptrap-v4.0.0.asciidoc[] include::snmptrap-v3.1.0.asciidoc[] include::snmptrap-v3.0.6.asciidoc[] include::snmptrap-v3.0.5.asciidoc[] diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.0.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.0.asciidoc new file mode 100644 index 000000000..cee29925a --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.0.asciidoc @@ -0,0 +1,392 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.0 +:release_date: 2024-05-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +//Uncomment after generated the initial versions of the target file +//include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +For migration information and guidelines, check out the migration guide. +**** + +// TODO: Add link to migration topic in the integration index. + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.1.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.1.asciidoc new file mode 100644 index 000000000..626bcad4e --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.1.asciidoc @@ -0,0 +1,403 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.1 +:release_date: 2024-05-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of behavioral and mapping differences between current stand-alone plugins and the new versions included in the `integration-snmp`. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.2.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.2.asciidoc new file mode 100644 index 000000000..03620f739 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.2.asciidoc @@ -0,0 +1,403 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.2 +:release_date: 2024-05-20 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of behavioral and mapping differences between current stand-alone plugins and the new versions included in the `integration-snmp`. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.3.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.3.asciidoc new file mode 100644 index 000000000..99a53d2c2 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.3.asciidoc @@ -0,0 +1,400 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.3 +:release_date: 2024-06-11 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of behavioral and mapping differences between current stand-alone plugins and the new versions included in the `integration-snmp`. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you install the new integration, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. +**** + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.4.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.4.asciidoc new file mode 100644 index 000000000..4a540309f --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.4.asciidoc @@ -0,0 +1,389 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.4 +:release_date: 2024-07-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.5.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.5.asciidoc new file mode 100644 index 000000000..fd4c7f81a --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.5.asciidoc @@ -0,0 +1,389 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.5 +:release_date: 2025-01-06 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.6.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.6.asciidoc new file mode 100644 index 000000000..7e7a12dad --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.6.asciidoc @@ -0,0 +1,389 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.6 +:release_date: 2025-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv` + * There is no default value for this setting + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.0.7.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.0.7.asciidoc new file mode 100644 index 000000000..87d9562e5 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.0.7.asciidoc @@ -0,0 +1,392 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.7 +:release_date: 2025-07-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: + - `noAuthNoPriv`: allows receiving traps messages without authentication or encryption. + - `authNoPriv`: trap messages must be authenticated according to <<{version}-plugins-{type}s-{plugin}-security_name>>/<<{version}-plugins-{type}s-{plugin}-auth_protocol>>/<<{version}-plugins-{type}s-{plugin}-auth_pass>>. Encrypted messages are allowed but not required. + - `authPriv`: trap messages must be both authenticated according to <<{version}-plugins-{type}s-{plugin}-security_name>>/<<{version}-plugins-{type}s-{plugin}-auth_protocol>>/<<{version}-plugins-{type}s-{plugin}-auth_pass>> and encrypted according to <<{version}-plugins-{type}s-{plugin}-priv_protocol>>/<<{version}-plugins-{type}s-{plugin}-priv_pass>>. + * The default value is `noAuthNoPriv`. + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/snmptrap-v4.1.0.asciidoc b/docs/versioned-plugins/inputs/snmptrap-v4.1.0.asciidoc new file mode 100644 index 000000000..57002f7b3 --- /dev/null +++ b/docs/versioned-plugins/inputs/snmptrap-v4.1.0.asciidoc @@ -0,0 +1,394 @@ +:integration: snmp +:plugin: snmptrap +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.0 +:release_date: 2025-08-18 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP trap input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +.Migrating to `logstash-integration-snmp` from stand-alone `input-snmptrap` +**** +The `logstash-input-snmptrap` plugin is now a component of the `logstash-integration-snmp` plugin which is +bundled with {ls} 8.15.0 by default. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +Before you upgrade to {ls} 8.15.0, be aware of link:{logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] +between current stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The `logstash-input-snmptrap` plugin reads SNMP trap messages as events. + +Resulting `message` field resembles: +[source,json] +{"agent_addr":"192.168.1.40", "generic_trap":6, "specific_trap":15511, "enterprise":"1.3.6.1.2.1.1.1", "variable_bindings":{"1.3.6.1.2.1.1.2.0":"test one", "1.3.6.1.2.1.1.1.0":"test two"}, "type":"V1TRAP", "community":"public", "version":1, "timestamp":1500} + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Event Metadata and the Elastic Common Schema (ECS) + +Because SNMP data has specific field names based on OIDs, we recommend setting a <<{version}-plugins-{type}s-{plugin}-target>>. +The source host field changes based on <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. + +[cols="> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-community>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mib_paths>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["default", "ruby_snmp", "dotted_string"]`|No +| <<{version}-plugins-{type}s-{plugin}-oid_map_field_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|Yes +| <<{version}-plugins-{type}s-{plugin}-oid_path_length>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-oid_root_skip>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-supported_transports>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-supported_versions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_provided_mibs>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-yamlmibdir>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +|======================================================================= + +==== SNMPv3 Authentication Options + +This plugin supports the following SNMPv3 authentication options. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-auth_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-auth_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["md5", "sha", "sha2", "hmac128sha224", "hmac192sha256", "hmac256sha384", "hmac384sha512"]`|No +| <<{version}-plugins-{type}s-{plugin}-priv_pass>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-priv_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["des", "3des", "aes", "aes128", "aes192", "aes256"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["noAuthNoPriv", "authNoPriv", "authPriv"]`|No +| <<{version}-plugins-{type}s-{plugin}-security_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +==== SNMP Trap Input Configuration Options + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +[id="{version}-plugins-{type}s-{plugin}-community"] +===== `community` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["public"]` + +The SNMPv1 and SNMPv2c communities to listen for. +To allow any community, set this config value to empty `community => []`. + +Examples + +*Listen for `public` and `guest` communities* +[source,ruby] +----- +input { + snmptrap { + community => ["public", "guest"] + } +} +----- + +*Listen for all communities* +[source,ruby] +----- +input { + snmptrap { + community => [] + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-mib_paths"] +===== `mib_paths` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The `mib_paths` option specifies the location of one or more imported MIB files. +The value can be either a dir path containing the imported MIB (`.dic`, `.yaml`) files or a +file path to a single MIB file. + +[id="{version}-plugins-{type}s-{plugin}-oid_mapping_format"] +===== `oid_mapping_format` + + * Value can be any of: `default`, `ruby_snmp`, `dotted_string` + * Default value is `"default"` + +Defines the mapping textual representation of an OID in the Logstash event: +* `default` translates every identifier, using the MIBs resolved names, separated by dots. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `iso.org.dod.internet.mgmt.mib-2.system.sysObjectID.0` +* `ruby_snmp` produces field names prefixed by the MIBs module name, followed by the latest resolved identifier name and unknowns values. Example: +`1.3.6.1.2.1.1.2.0` is mapped as `SNMPv2-MIB::sysObjectID.0`. +* `dotted_string` maps fields using the standard dotted string representation, Example: +`1.3.6.1.2.1.1.2.0` is mapped as `1.3.6.1.2.1.1.2.0` + +[id="{version}-plugins-{type}s-{plugin}-oid_map_field_values"] +===== `oid_map_field_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Defines if the Logstash event fields values, which types are `OID`, are mapped using the configured OID textual representation +set on the <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> option. + +[id="{version}-plugins-{type}s-{plugin}-oid_root_skip"] +===== `oid_root_skip` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5` +which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_path_length>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-oid_path_length"] +===== `oid_path_length` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The `oid_path_length` option specifies the number of OID root digits to retain in the event field name. +For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2` +which would result in a field name "1.0". Similarly when a MIB is used an OID such +"1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0" + +[NOTE] +* You can use this setting or <<{version}-plugins-{type}s-{plugin}-oid_root_skip>>, but not both at the same time. +* Use this setting only if <<{version}-plugins-{type}s-{plugin}-oid_mapping_format>> is set to `default`. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1062` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. hence the default of 1062. + +[id="{version}-plugins-{type}s-{plugin}-supported_transports"] +===== `supported_transports` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `tcp`, `udp` + * Default value is `["udp"]` + +The supported transport protocols to listen on. + +SNMP was originally designed for use with UDP as transport protocol and is the official recommendation. +TCP is an optional transport mapping and can be enabled if needed. For more details +on SNMP over TCP, please refer to the link:https://datatracker.ietf.org/doc/html/rfc3430[RFC-3430]. + +[id="{version}-plugins-{type}s-{plugin}-supported_versions"] +===== `supported_versions` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `1`, `2c`, `3` + * Default value is `["1", "2c"]` + +The supported SNMP protocol versions to listen on. +SNMP messages for versions that are either unsupported and/or disabled are automatically discarded. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The name of the field under which SNMP payloads are assigned. +If not specified data will be stored in the root of the event. + +Setting a target is recommended when <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled. + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is 75% of the number of CPU cores + +The number of threads to use for processing the received SNMP trap messages. + +[id="{version}-plugins-{type}s-{plugin}-use_provided_mibs"] +===== `use_provided_mibs` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +This plugin provides all IETF MIBs (management information bases), publicly available in the link:https://www.ibr.cs.tu-bs.de/projects/libsmi[libsmi] version `0.5.0`. +When enabled, it automatically loads the bundled MIBs and provides mapping of the numeric OIDs to MIB field names in the resulting event. + +[id="{version}-plugins-{type}s-{plugin}-yamlmibdir"] +===== `yamlmibdir` +deprecated[4.0.0, Replaced by <<{version}-plugins-{type}s-{plugin}-mib_paths>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +directory of YAML MIB maps (same format ruby-snmp uses) + +==== SNMPv3 Authentication Options + +A **single user** can be configured. Multiple snmptrap input declarations will be needed if multiple SNMPv3 users are required. +These options are required only if you are using SNMPv3. + +[id="{version}-plugins-{type}s-{plugin}-auth_pass"] +===== `auth_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `auth_pass` option specifies the SNMPv3 authentication passphrase or password. + +[id="{version}-plugins-{type}s-{plugin}-auth_protocol"] +===== `auth_protocol` + +The `auth_protocol` option specifies the SNMPv3 authentication protocol or type + +* Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512` +* Note that `sha2` and `hmac192sha256` are equivalent +* There is no default value for this setting + +[id="{version}-plugins-{type}s-{plugin}-priv_pass"] +===== `priv_pass` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting + +The `priv_pass` option specifies the SNMPv3 encryption password. + +[id="{version}-plugins-{type}s-{plugin}-priv_protocol"] +===== `priv_protocol` + + * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256` + * Note that `aes` and `aes128` are equivalent + * There is no default value for this setting + +The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol. + +[id="{version}-plugins-{type}s-{plugin}-security_level"] +===== `security_level` + + * Value can be any of: + - `noAuthNoPriv`: allows receiving traps messages without authentication or encryption. + - `authNoPriv`: trap messages must be authenticated according to <<{version}-plugins-{type}s-{plugin}-security_name>>/<<{version}-plugins-{type}s-{plugin}-auth_protocol>>/<<{version}-plugins-{type}s-{plugin}-auth_pass>>. Encrypted messages are allowed but not required. + - `authPriv`: trap messages must be both authenticated according to <<{version}-plugins-{type}s-{plugin}-security_name>>/<<{version}-plugins-{type}s-{plugin}-auth_protocol>>/<<{version}-plugins-{type}s-{plugin}-auth_pass>> and encrypted according to <<{version}-plugins-{type}s-{plugin}-priv_protocol>>/<<{version}-plugins-{type}s-{plugin}-priv_pass>>. + * The default value is `noAuthNoPriv`. + +The `security_level` option specifies the SNMPv3 security level between +Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. + +[id="{version}-plugins-{type}s-{plugin}-security_name"] +===== `security_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The `security_name` option specifies the SNMPv3 security name or user name. + +==== Configuration examples + +*Specifying SNMPv3 traps settings* + +[source,ruby] +----- +input { + snmptrap { + supported_versions => ['3'] + security_name => "mySecurityName" + auth_protocol => "sha" + auth_pass => "ShaPassword" + priv_protocol => "aes" + priv_pass => "AesPasword" + security_level => "authPriv" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/sqs-index.asciidoc b/docs/versioned-plugins/inputs/sqs-index.asciidoc index 40b451dc4..4ffb7efc0 100644 --- a/docs/versioned-plugins/inputs/sqs-index.asciidoc +++ b/docs/versioned-plugins/inputs/sqs-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -28,6 +32,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-04 |======================================================================= +include::sqs-v7.2.1.asciidoc[] +include::sqs-v7.2.0.asciidoc[] +include::sqs-v7.1.8.asciidoc[] +include::sqs-v7.1.7.asciidoc[] include::sqs-v7.1.6.asciidoc[] include::sqs-v7.1.5.asciidoc[] include::sqs-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/inputs/sqs-v7.1.7.asciidoc b/docs/versioned-plugins/inputs/sqs-v7.1.7.asciidoc new file mode 100644 index 000000000..41dfdac43 --- /dev/null +++ b/docs/versioned-plugins/inputs/sqs-v7.1.7.asciidoc @@ -0,0 +1,296 @@ +:integration: aws +:plugin: sqs +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + + +Pull events from an Amazon Web Services Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the +Amazon Web Services suite of tools. + +Although SQS is similar to other queuing systems like AMQP, it +uses a custom API and requires that you have an AWS account. +See http://aws.amazon.com/sqs/ for more details on how SQS works, +what the pricing schedule looks like and how to setup a queue. + +To use this plugin, you *must*: + + * Have an AWS account + * Setup an SQS queue + * Create an identity that has access to consume messages from the queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:ChangeMessageVisibility` + * `sqs:ChangeMessageVisibilityBatch` + * `sqs:DeleteMessage` + * `sqs:DeleteMessageBatch` + * `sqs:GetQueueAttributes` + * `sqs:GetQueueUrl` + * `sqs:ListQueues` + * `sqs:ReceiveMessage` + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy is as follows: +[source,json] + { + "Statement": [ + { + "Action": [ + "sqs:ChangeMessageVisibility", + "sqs:ChangeMessageVisibilityBatch", + "sqs:DeleteMessage", + "sqs:DeleteMessageBatch", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:sqs:us-east-1:123456789012:Logstash" + ] + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-id_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-md5_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-polling_frequency>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sent_timestamp_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + +* Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] +* Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to SQS. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + sqs { + access_key_id => "1234" + secret_access_key => "secret" + queue => "logstash-test-queue" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-id_field"] +===== `id_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message ID + +[id="{version}-plugins-{type}s-{plugin}-md5_field"] +===== `md5_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message MD5 checksum + +[id="{version}-plugins-{type}s-{plugin}-polling_frequency"] +===== `polling_frequency` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +Polling frequency, default is 20 seconds + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +ID of the AWS account owning the queue if you want to use a https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-basic-examples-of-sqs-policies.html#grant-two-permissions-to-one-account[cross-account queue] with embedded policy. Note that AWS SDK only support numerical account ID and not account aliases. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-sent_timestamp_field"] +===== `sent_timestamp_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message Sent Timestamp + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/sqs-v7.1.8.asciidoc b/docs/versioned-plugins/inputs/sqs-v7.1.8.asciidoc new file mode 100644 index 000000000..26355c0f3 --- /dev/null +++ b/docs/versioned-plugins/inputs/sqs-v7.1.8.asciidoc @@ -0,0 +1,296 @@ +:integration: aws +:plugin: sqs +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + + +Pull events from an Amazon Web Services Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the +Amazon Web Services suite of tools. + +Although SQS is similar to other queuing systems like AMQP, it +uses a custom API and requires that you have an AWS account. +See http://aws.amazon.com/sqs/ for more details on how SQS works, +what the pricing schedule looks like and how to setup a queue. + +To use this plugin, you *must*: + + * Have an AWS account + * Setup an SQS queue + * Create an identity that has access to consume messages from the queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:ChangeMessageVisibility` + * `sqs:ChangeMessageVisibilityBatch` + * `sqs:DeleteMessage` + * `sqs:DeleteMessageBatch` + * `sqs:GetQueueAttributes` + * `sqs:GetQueueUrl` + * `sqs:ListQueues` + * `sqs:ReceiveMessage` + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy is as follows: +[source,json] + { + "Statement": [ + { + "Action": [ + "sqs:ChangeMessageVisibility", + "sqs:ChangeMessageVisibilityBatch", + "sqs:DeleteMessage", + "sqs:DeleteMessageBatch", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:sqs:us-east-1:123456789012:Logstash" + ] + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-id_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-md5_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-polling_frequency>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sent_timestamp_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + +* Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] +* Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to SQS. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + sqs { + access_key_id => "1234" + secret_access_key => "secret" + queue => "logstash-test-queue" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-id_field"] +===== `id_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message ID + +[id="{version}-plugins-{type}s-{plugin}-md5_field"] +===== `md5_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message MD5 checksum + +[id="{version}-plugins-{type}s-{plugin}-polling_frequency"] +===== `polling_frequency` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +Polling frequency, default is 20 seconds + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +ID of the AWS account owning the queue if you want to use a https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-basic-examples-of-sqs-policies.html#grant-two-permissions-to-one-account[cross-account queue] with embedded policy. Note that AWS SDK only support numerical account ID and not account aliases. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-sent_timestamp_field"] +===== `sent_timestamp_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message Sent Timestamp + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/sqs-v7.2.0.asciidoc b/docs/versioned-plugins/inputs/sqs-v7.2.0.asciidoc new file mode 100644 index 000000000..697b2b9b2 --- /dev/null +++ b/docs/versioned-plugins/inputs/sqs-v7.2.0.asciidoc @@ -0,0 +1,308 @@ +:integration: aws +:plugin: sqs +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + + +Pull events from an Amazon Web Services Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the +Amazon Web Services suite of tools. + +Although SQS is similar to other queuing systems like AMQP, it +uses a custom API and requires that you have an AWS account. +See http://aws.amazon.com/sqs/ for more details on how SQS works, +what the pricing schedule looks like and how to setup a queue. + +To use this plugin, you *must*: + + * Have an AWS account + * Setup an SQS queue + * Create an identity that has access to consume messages from the queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:ChangeMessageVisibility` + * `sqs:ChangeMessageVisibilityBatch` + * `sqs:DeleteMessage` + * `sqs:DeleteMessageBatch` + * `sqs:GetQueueAttributes` + * `sqs:GetQueueUrl` + * `sqs:ListQueues` + * `sqs:ReceiveMessage` + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy is as follows: +[source,json] + { + "Statement": [ + { + "Action": [ + "sqs:ChangeMessageVisibility", + "sqs:ChangeMessageVisibilityBatch", + "sqs:DeleteMessage", + "sqs:DeleteMessageBatch", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:sqs:us-east-1:123456789012:Logstash" + ] + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-id_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-md5_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-polling_frequency>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sent_timestamp_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + +* Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] +* Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to SQS. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + sqs { + access_key_id => "1234" + secret_access_key => "secret" + queue => "logstash-test-queue" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-id_field"] +===== `id_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message ID + +[id="{version}-plugins-{type}s-{plugin}-md5_field"] +===== `md5_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message MD5 checksum + +[id="{version}-plugins-{type}s-{plugin}-polling_frequency"] +===== `polling_frequency` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +Polling frequency, default is 20 seconds + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +ID of the AWS account owning the queue if you want to use a https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-basic-examples-of-sqs-policies.html#grant-two-permissions-to-one-account[cross-account queue] with embedded policy. Note that AWS SDK only support numerical account ID and not account aliases. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-sent_timestamp_field"] +===== `sent_timestamp_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message Sent Timestamp + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/sqs-v7.2.1.asciidoc b/docs/versioned-plugins/inputs/sqs-v7.2.1.asciidoc new file mode 100644 index 000000000..b2086f228 --- /dev/null +++ b/docs/versioned-plugins/inputs/sqs-v7.2.1.asciidoc @@ -0,0 +1,308 @@ +:integration: aws +:plugin: sqs +:type: input +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + + +Pull events from an Amazon Web Services Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the +Amazon Web Services suite of tools. + +Although SQS is similar to other queuing systems like AMQP, it +uses a custom API and requires that you have an AWS account. +See http://aws.amazon.com/sqs/ for more details on how SQS works, +what the pricing schedule looks like and how to setup a queue. + +To use this plugin, you *must*: + + * Have an AWS account + * Setup an SQS queue + * Create an identity that has access to consume messages from the queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:ChangeMessageVisibility` + * `sqs:ChangeMessageVisibilityBatch` + * `sqs:DeleteMessage` + * `sqs:DeleteMessageBatch` + * `sqs:GetQueueAttributes` + * `sqs:GetQueueUrl` + * `sqs:ListQueues` + * `sqs:ReceiveMessage` + +Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user. +A sample policy is as follows: +[source,json] + { + "Statement": [ + { + "Action": [ + "sqs:ChangeMessageVisibility", + "sqs:ChangeMessageVisibilityBatch", + "sqs:DeleteMessage", + "sqs:DeleteMessageBatch", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:sqs:us-east-1:123456789012:Logstash" + ] + } + ] + } + +See http://aws.amazon.com/iam/ for more details on setting up AWS identities. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-id_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-md5_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-polling_frequency>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sent_timestamp_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-threads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + +* Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] +* Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to SQS. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + input { + sqs { + access_key_id => "1234" + secret_access_key => "secret" + queue => "logstash-test-queue" + additional_settings => { + force_path_style => true + follow_redirects => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-id_field"] +===== `id_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message ID + +[id="{version}-plugins-{type}s-{plugin}-md5_field"] +===== `md5_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message MD5 checksum + +[id="{version}-plugins-{type}s-{plugin}-polling_frequency"] +===== `polling_frequency` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +Polling frequency, default is 20 seconds + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +ID of the AWS account owning the queue if you want to use a https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-basic-examples-of-sqs-policies.html#grant-two-permissions-to-one-account[cross-account queue] with embedded policy. Note that AWS SDK only support numerical account ID and not account aliases. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-sent_timestamp_field"] +===== `sent_timestamp_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Name of the event field in which to store the SQS message Sent Timestamp + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-threads"] +===== `threads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/syslog-index.asciidoc b/docs/versioned-plugins/inputs/syslog-index.asciidoc index c065923be..385372ed2 100644 --- a/docs/versioned-plugins/inputs/syslog-index.asciidoc +++ b/docs/versioned-plugins/inputs/syslog-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-03-04 | <> | 2023-10-17 | <> | 2021-11-11 | <> | 2021-03-22 @@ -21,6 +22,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::syslog-v3.7.1.asciidoc[] include::syslog-v3.7.0.asciidoc[] include::syslog-v3.6.0.asciidoc[] include::syslog-v3.5.0.asciidoc[] diff --git a/docs/versioned-plugins/inputs/syslog-v3.7.1.asciidoc b/docs/versioned-plugins/inputs/syslog-v3.7.1.asciidoc new file mode 100644 index 000000000..4e10f23dc --- /dev/null +++ b/docs/versioned-plugins/inputs/syslog-v3.7.1.asciidoc @@ -0,0 +1,220 @@ +:plugin: syslog +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.7.1 +:release_date: 2025-03-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-syslog/blob/v3.7.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Syslog input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read syslog messages as events over the network. + +This input is a good choice if you already use syslog today. +It is also a good choice if you want to receive logs from +appliances and network devices where you cannot run your own +log collector. + +Of course, 'syslog' is a very muddy term. By default, this input only +supports `RFC3164` syslog with some small modifications. However, some +non-standard syslog formats can be read and parsed if a functional +`grok_pattern` is provided. The date format is still only allowed to be +`RFC3164` style or `ISO8601`. + +For more information see the http://www.ietf.org/rfc/rfc3164.txt[RFC3164 page]. + +Note: This input will start listeners on both TCP and UDP. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Syslog Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-facility_labels>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-grok_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-locale>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-severity_labels>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-syslog_field>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_labels>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (for example, `priority` for syslog priority) + ** `v1`,`v8`: uses fields that are compatible with Elastic Common Schema (for example, `[log][syslog][priority]`) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. + +[id="{version}-plugins-{type}s-{plugin}-facility_labels"] +===== `facility_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["kernel", "user-level", "mail", "system", "security/authorization", "syslogd", "line printer", "network news", "UUCP", "clock", "security/authorization", "FTP", "NTP", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7"]` + +Labels for facility levels defined in RFC3164. + +You can use this option to override the integer->label mapping for syslog inputs +that behave differently than the RFCs. + +Provide a zero-indexed array with all of your facility labels _in order_. +If a log message contains a facility number with no corresponding entry, +the facility_label is not added to the event. + +[id="{version}-plugins-{type}s-{plugin}-grok_pattern"] +===== `grok_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"<%{POSINT:priority}>%{SYSLOGLINE}"` + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"<%{POSINT:priority}>%{SYSLOGLINE}"` + ** ECS Compatibility enabled: `"<%{POSINT:[log][syslog][priority]:int}>%{SYSLOGLINE}"` + +The default value should read and properly parse syslog lines which are +fully compliant with http://www.ietf.org/rfc/rfc3164.txt[RFC3164]. + +You can override this value to parse non-standard lines with a valid grok +pattern which will parse the received lines. If the line is unable to +be parsed, the `_grokparsefailure_sysloginput` tag will be added. + +The grok pattern must provide a `timestamp` field. If the `timestamp` +field is omitted, or is unable to be parsed as `RFC3164` style or +`ISO8601`, a `_dateparsefailure` tag will be added. + + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to listen on. + +[id="{version}-plugins-{type}s-{plugin}-locale"] +===== `locale` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag. +Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX. +If not specified, the platform default will be used. + +The locale is mostly necessary to be set for parsing month names (pattern with MMM) and +weekday names (pattern with EEE). + + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `514` + +The port to listen on. Remember that ports less than 1024 (privileged +ports) may require root to use. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-severity_labels"] +===== `severity_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug"]` + +Labels for severity levels defined in RFC3164. + +Provide a zero-indexed array with all of your severity labels _in order_. +If a log message contains a severity label with no corresponding entry, +the severity_label is not added to the event. + +[id="{version}-plugins-{type}s-{plugin}-syslog_field"] +===== `syslog_field` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"message"` + +Codecs process the data before the rest of the data is parsed. Some codecs, +like CEF, put the syslog data into another field after pre-processing the +data. Use this option in conjunction with the `grok_pattern` configuration +to allow the syslog input plugin to fully parse the syslog data in this case. + +[source,sh] +------- +input { + syslog { + port => 12345 + codec => cef + syslog_field => "syslog" + grok_pattern => "<%{POSINT:priority}>%{SYSLOGTIMESTAMP:timestamp} CUSTOM GROK HERE" + } +} +------- + +[id="{version}-plugins-{type}s-{plugin}-timezone"] +===== `timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Specify a time zone canonical ID to be used for date parsing. +The valid IDs are listed on the [Joda.org available time zones page](http://joda-time.sourceforge.net/timezones.html). +This is useful in case the time zone cannot be extracted from the value, +and is not the platform default. +If this is not specified the platform default will be used. +Canonical ID is good as it takes care of daylight saving time for you. +For example, `America/Los_Angeles` or `Europe/Paris` are valid IDs. + +[id="{version}-plugins-{type}s-{plugin}-use_labels"] +===== `use_labels` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Use label parsing for severity and facility levels. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-index.asciidoc b/docs/versioned-plugins/inputs/tcp-index.asciidoc index b6e4187fd..b68facbd4 100644 --- a/docs/versioned-plugins/inputs/tcp-index.asciidoc +++ b/docs/versioned-plugins/inputs/tcp-index.asciidoc @@ -5,6 +5,16 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-09-04 +| <> | 2025-02-12 +| <> | 2025-02-04 +| <> | 2025-01-10 +| <> | 2025-09-04 +| <> | 2025-02-12 +| <> | 2025-02-04 +| <> | 2024-11-18 +| <> | 2024-10-16 +| <> | 2024-04-23 | <> | 2023-10-18 | <> | 2023-07-06 | <> | 2023-06-29 @@ -59,6 +69,16 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::tcp-v7.0.3.asciidoc[] +include::tcp-v7.0.2.asciidoc[] +include::tcp-v7.0.1.asciidoc[] +include::tcp-v7.0.0.asciidoc[] +include::tcp-v6.4.7.asciidoc[] +include::tcp-v6.4.6.asciidoc[] +include::tcp-v6.4.5.asciidoc[] +include::tcp-v6.4.4.asciidoc[] +include::tcp-v6.4.3.asciidoc[] +include::tcp-v6.4.2.asciidoc[] include::tcp-v6.4.1.asciidoc[] include::tcp-v6.4.0.asciidoc[] include::tcp-v6.3.5.asciidoc[] diff --git a/docs/versioned-plugins/inputs/tcp-v6.4.2.asciidoc b/docs/versioned-plugins/inputs/tcp-v6.4.2.asciidoc new file mode 100644 index 000000000..c800365d3 --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v6.4.2.asciidoc @@ -0,0 +1,370 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.4.2 +:release_date: 2024-04-23 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v6.4.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the connecting clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v6.4.3.asciidoc b/docs/versioned-plugins/inputs/tcp-v6.4.3.asciidoc new file mode 100644 index 000000000..ca431113b --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v6.4.3.asciidoc @@ -0,0 +1,370 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.4.3 +:release_date: 2024-10-16 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v6.4.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the connecting clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v6.4.4.asciidoc b/docs/versioned-plugins/inputs/tcp-v6.4.4.asciidoc new file mode 100644 index 000000000..a1086250a --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v6.4.4.asciidoc @@ -0,0 +1,370 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.4.4 +:release_date: 2024-11-18 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v6.4.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the connecting clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v6.4.5.asciidoc b/docs/versioned-plugins/inputs/tcp-v6.4.5.asciidoc new file mode 100644 index 000000000..f17dd093f --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v6.4.5.asciidoc @@ -0,0 +1,370 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.4.5 +:release_date: 2025-02-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v6.4.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the connecting clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v6.4.6.asciidoc b/docs/versioned-plugins/inputs/tcp-v6.4.6.asciidoc new file mode 100644 index 000000000..d42431432 --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v6.4.6.asciidoc @@ -0,0 +1,370 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.4.6 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v6.4.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the connecting clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v6.4.7.asciidoc b/docs/versioned-plugins/inputs/tcp-v6.4.7.asciidoc new file mode 100644 index 000000000..c8f11cfe8 --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v6.4.7.asciidoc @@ -0,0 +1,370 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.4.7 +:release_date: 2025-09-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v6.4.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the connecting clients. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.4.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v7.0.0.asciidoc b/docs/versioned-plugins/inputs/tcp-v7.0.0.asciidoc new file mode 100644 index 000000000..b37270af1 --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v7.0.0.asciidoc @@ -0,0 +1,358 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.0 +:release_date: 2025-01-10 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v7.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== TCP Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ssl_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| ssl_enable |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> in `server` mode and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> in `client` mode +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v7.0.1.asciidoc b/docs/versioned-plugins/inputs/tcp-v7.0.1.asciidoc new file mode 100644 index 000000000..e2ac70397 --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v7.0.1.asciidoc @@ -0,0 +1,358 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.1 +:release_date: 2025-02-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v7.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== TCP Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ssl_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| ssl_enable |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> in `server` mode and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> in `client` mode +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v7.0.2.asciidoc b/docs/versioned-plugins/inputs/tcp-v7.0.2.asciidoc new file mode 100644 index 000000000..619ce79ac --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v7.0.2.asciidoc @@ -0,0 +1,358 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.2 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v7.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== TCP Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ssl_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| ssl_enable |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> in `server` mode and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> in `client` mode +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/inputs/tcp-v7.0.3.asciidoc b/docs/versioned-plugins/inputs/tcp-v7.0.3.asciidoc new file mode 100644 index 000000000..7bed702f0 --- /dev/null +++ b/docs/versioned-plugins/inputs/tcp-v7.0.3.asciidoc @@ -0,0 +1,358 @@ +:plugin: tcp +:type: input +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.3 +:release_date: 2025-09-04 +:changelog_url: https://github.com/logstash-plugins/logstash-input-tcp/blob/v7.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp input plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a TCP socket. + +Like stdin and file inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +===== Accepting log4j2 logs + +Log4j2 can send JSON over a socket, and we can use that combined with our tcp +input to accept the logs. + +First, we need to configure your application to send logs in JSON over a +socket. The following log4j2.xml accomplishes this task. + +Note, you will want to change the `host` and `port` settings in this +configuration to match your needs. + + + + + + + + + + + + + + +To accept this in Logstash, you will want tcp input and a date filter: + + input { + tcp { + port => 12345 + codec => json + } + } + +and add a date filter to take log4j2's `timeMillis` field and use it as the +event timestamp + + filter { + date { + match => [ "timeMillis", "UNIX_MS" ] + } + } + +[id="{version}-plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +In addition to decoding the events, this input will add metadata about the TCP connection itself to each event. +This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves. + +Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream. +With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +[cols="3,7,5"] +|======================================================================= +| Metadata Group | ecs: `v1`, `v8` | ecs: `disabled` + +.3+|Source Metadata from the TCP connection +on which events are being received, including +the sender's name, ip, and outbound port. l|[@metadata][input][tcp][source][name] l|[host] +l|[@metadata][input][tcp][source][ip] l|[@metadata][ip_address] +l|[@metadata][input][tcp][source][port] l|[port] + +.2+|Proxy Metadata from a proxied TCP connection. +Available when receiving events by proxy and +`proxy_protocol => true` l|[@metadata][input][tcp][proxy][ip] l|[proxy_host] +l|[@metadata][input][tcp][proxy][port] l|[proxy_port] + +.1+|SSL Subject Metadata from a secured TCP +connection. Available when `ssl_enabled => true` +AND `ssl_client_authentication => 'optional' or 'required'` l|[@metadata][input][tcp][ssl][subject] l|[sslsubject] +|======================================================================= + +For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened. +If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event: + +[source,txt] +----- +filter { + if [@metadata][input][tcp][source] and ![host] { + mutate { + copy => { + "[@metadata][input][tcp][source][name]" => "[host][name]" + "[@metadata][input][tcp][source][ip]" => "[host][ip]" + } + } + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-proxy_protocol>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-tcp_keep_alive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"] +===== `dns_reverse_lookup_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled, +the address metadata that is added to events will contain the source address as-specified +at the TCP layer and IPs will not be resolved to hostnames. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: + ** `disabled`: unstructured connection metadata added at root level + ** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]` + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)]. +The value of this setting affects the <<{version}-plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-proxy_protocol"] +===== `proxy_protocol` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Proxy protocol support, only v1 is supported at this time +http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files or paths. All the certificates will be read and added to the trust store. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value includes _all_ cipher suites enabled by the JDK and depends on JDK configuration + +Supported cipher suites vary depending on Java version used, and entries look like `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`. +For more information, see Oracle’s https://docs.oracle.com/en/java/javase/11/security/oracle-providers.html#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2[JDK SunJSSE provider documentation] and +the table of supported https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#jsse-cipher-suite-names[Java cipher suite names]. + +NOTE: To check the supported cipher suites locally run the following script: `$LS_HOME/bin/ruby -e 'p javax.net.ssl.SSLServerSocketFactory.getDefault.getSupportedCipherSuites'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `required` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_extra_chain_certs"] +===== `ssl_extra_chain_certs` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +An Array of paths to extra X509 certificates. +These are used together with the certificate to construct the certificate chain +presented to the client. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The path to the private key corresponding to the specified certificate (PEM format). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase for the private key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-tcp_keep_alive"] +===== `tcp_keep_alive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Instruct the socket to use TCP keep alive. If it's `true` then the underlying socket +will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any +keep alive setting for the underlying socket. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== TCP Input Obsolete Configuration Options + +WARNING: As of version `7.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ssl_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| ssl_enable |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> in `server` mode and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> in `client` mode +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/integrations-index.asciidoc b/docs/versioned-plugins/integrations-index.asciidoc index c729ab836..b856b6258 100644 --- a/docs/versioned-plugins/integrations-index.asciidoc +++ b/docs/versioned-plugins/integrations-index.asciidoc @@ -15,3 +15,5 @@ include::integrations/logstash-index.asciidoc[] include::integrations/rabbitmq-index.asciidoc[] +include::integrations/snmp-index.asciidoc[] + diff --git a/docs/versioned-plugins/integrations/aws-index.asciidoc b/docs/versioned-plugins/integrations/aws-index.asciidoc index 8038936fc..54953cb89 100644 --- a/docs/versioned-plugins/integrations/aws-index.asciidoc +++ b/docs/versioned-plugins/integrations/aws-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -16,6 +20,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2022-07-25 |======================================================================= +include::aws-v7.2.1.asciidoc[] +include::aws-v7.2.0.asciidoc[] +include::aws-v7.1.8.asciidoc[] +include::aws-v7.1.7.asciidoc[] include::aws-v7.1.6.asciidoc[] include::aws-v7.1.5.asciidoc[] include::aws-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/integrations/aws-v7.1.7.asciidoc b/docs/versioned-plugins/integrations/aws-v7.1.7.asciidoc new file mode 100644 index 000000000..4ecd6081e --- /dev/null +++ b/docs/versioned-plugins/integrations/aws-v7.1.7.asciidoc @@ -0,0 +1,36 @@ +:plugin: aws +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== AWS Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The AWS Integration Plugin provides integrated plugins for working with Amazon Web Services: + + * {logstash-ref}/plugins-codecs-cloudfront.html[Cloudfront Codec Plugin] + * {logstash-ref}/plugins-codecs-cloudtrail.html[Cloudtrail Codec Plugin] + * {logstash-ref}/plugins-inputs-cloudwatch.html[Cloudwatch Input Plugin] + * {logstash-ref}/plugins-inputs-s3.html[S3 Input Plugin] + * {logstash-ref}/plugins-inputs-sqs.html[Sqs Input Plugin] + * {logstash-ref}/plugins-outputs-cloudwatch.html[Cloudwatch Output Plugin] + * {logstash-ref}/plugins-outputs-s3.html[S3 Output Plugin] + * {logstash-ref}/plugins-outputs-sns.html[Sns Output Plugin] + * {logstash-ref}/plugins-outputs-sqs.html[Sqs Output Plugin] + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/aws-v7.1.8.asciidoc b/docs/versioned-plugins/integrations/aws-v7.1.8.asciidoc new file mode 100644 index 000000000..5425f61b9 --- /dev/null +++ b/docs/versioned-plugins/integrations/aws-v7.1.8.asciidoc @@ -0,0 +1,36 @@ +:plugin: aws +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== AWS Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The AWS Integration Plugin provides integrated plugins for working with Amazon Web Services: + + * {logstash-ref}/plugins-codecs-cloudfront.html[Cloudfront Codec Plugin] + * {logstash-ref}/plugins-codecs-cloudtrail.html[Cloudtrail Codec Plugin] + * {logstash-ref}/plugins-inputs-cloudwatch.html[Cloudwatch Input Plugin] + * {logstash-ref}/plugins-inputs-s3.html[S3 Input Plugin] + * {logstash-ref}/plugins-inputs-sqs.html[Sqs Input Plugin] + * {logstash-ref}/plugins-outputs-cloudwatch.html[Cloudwatch Output Plugin] + * {logstash-ref}/plugins-outputs-s3.html[S3 Output Plugin] + * {logstash-ref}/plugins-outputs-sns.html[Sns Output Plugin] + * {logstash-ref}/plugins-outputs-sqs.html[Sqs Output Plugin] + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/aws-v7.2.0.asciidoc b/docs/versioned-plugins/integrations/aws-v7.2.0.asciidoc new file mode 100644 index 000000000..629902a4f --- /dev/null +++ b/docs/versioned-plugins/integrations/aws-v7.2.0.asciidoc @@ -0,0 +1,36 @@ +:plugin: aws +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== AWS Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The AWS Integration Plugin provides integrated plugins for working with Amazon Web Services: + + * {logstash-ref}/plugins-codecs-cloudfront.html[Cloudfront Codec Plugin] + * {logstash-ref}/plugins-codecs-cloudtrail.html[Cloudtrail Codec Plugin] + * {logstash-ref}/plugins-inputs-cloudwatch.html[Cloudwatch Input Plugin] + * {logstash-ref}/plugins-inputs-s3.html[S3 Input Plugin] + * {logstash-ref}/plugins-inputs-sqs.html[Sqs Input Plugin] + * {logstash-ref}/plugins-outputs-cloudwatch.html[Cloudwatch Output Plugin] + * {logstash-ref}/plugins-outputs-s3.html[S3 Output Plugin] + * {logstash-ref}/plugins-outputs-sns.html[Sns Output Plugin] + * {logstash-ref}/plugins-outputs-sqs.html[Sqs Output Plugin] + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/aws-v7.2.1.asciidoc b/docs/versioned-plugins/integrations/aws-v7.2.1.asciidoc new file mode 100644 index 000000000..7830e9960 --- /dev/null +++ b/docs/versioned-plugins/integrations/aws-v7.2.1.asciidoc @@ -0,0 +1,36 @@ +:plugin: aws +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== AWS Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The AWS Integration Plugin provides integrated plugins for working with Amazon Web Services: + + * {logstash-ref}/plugins-codecs-cloudfront.html[Cloudfront Codec Plugin] + * {logstash-ref}/plugins-codecs-cloudtrail.html[Cloudtrail Codec Plugin] + * {logstash-ref}/plugins-inputs-cloudwatch.html[Cloudwatch Input Plugin] + * {logstash-ref}/plugins-inputs-s3.html[S3 Input Plugin] + * {logstash-ref}/plugins-inputs-sqs.html[Sqs Input Plugin] + * {logstash-ref}/plugins-outputs-cloudwatch.html[Cloudwatch Output Plugin] + * {logstash-ref}/plugins-outputs-s3.html[S3 Output Plugin] + * {logstash-ref}/plugins-outputs-sns.html[Sns Output Plugin] + * {logstash-ref}/plugins-outputs-sqs.html[Sqs Output Plugin] + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/elastic_enterprise_search-index.asciidoc b/docs/versioned-plugins/integrations/elastic_enterprise_search-index.asciidoc index dc7f7d927..de1c4ea05 100644 --- a/docs/versioned-plugins/integrations/elastic_enterprise_search-index.asciidoc +++ b/docs/versioned-plugins/integrations/elastic_enterprise_search-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-01-03 | <> | 2023-11-07 | <> | 2022-01-28 | <> | 2022-01-24 @@ -14,6 +15,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2021-05-05 |======================================================================= +include::elastic_enterprise_search-v3.0.1.asciidoc[] include::elastic_enterprise_search-v3.0.0.asciidoc[] include::elastic_enterprise_search-v2.2.1.asciidoc[] include::elastic_enterprise_search-v2.2.0.asciidoc[] diff --git a/docs/versioned-plugins/integrations/elastic_enterprise_search-v3.0.1.asciidoc b/docs/versioned-plugins/integrations/elastic_enterprise_search-v3.0.1.asciidoc new file mode 100644 index 000000000..241ba777e --- /dev/null +++ b/docs/versioned-plugins/integrations/elastic_enterprise_search-v3.0.1.asciidoc @@ -0,0 +1,30 @@ +:plugin: elastic_enterprise_search +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.0.1 +:release_date: 2025-01-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/blob/v3.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elastic Enterprise Search integration plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Elastic Enterprise Search integration plugin provides integrated plugins for working with https://www.elastic.co/app-search[Elastic App Search] and https://www.elastic.co/workplace-search[Elastic Workplace Search] services: + +* {logstash-ref}/plugins-outputs-elastic_app_search.html[Elastic App Search output plugin] +* {logstash-ref}/plugins-outputs-elastic_workplace_search.html[Elastic Workplace Search output plugin] + + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/jdbc-index.asciidoc b/docs/versioned-plugins/integrations/jdbc-index.asciidoc index 9695c39d2..c8f68e0e1 100644 --- a/docs/versioned-plugins/integrations/jdbc-index.asciidoc +++ b/docs/versioned-plugins/integrations/jdbc-index.asciidoc @@ -5,6 +5,16 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-05-30 +| <> | 2025-03-07 +| <> | 2024-12-23 +| <> | 2024-12-03 +| <> | 2024-08-09 +| <> | 2024-05-23 +| <> | 2024-05-08 +| <> | 2024-03-21 +| <> | 2024-02-13 +| <> | 2024-01-31 | <> | 2023-10-03 | <> | 2023-09-04 | <> | 2023-06-15 @@ -40,6 +50,16 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2020-01-09 |======================================================================= +include::jdbc-v5.6.0.asciidoc[] +include::jdbc-v5.5.3.asciidoc[] +include::jdbc-v5.5.2.asciidoc[] +include::jdbc-v5.5.1.asciidoc[] +include::jdbc-v5.5.0.asciidoc[] +include::jdbc-v5.4.11.asciidoc[] +include::jdbc-v5.4.10.asciidoc[] +include::jdbc-v5.4.9.asciidoc[] +include::jdbc-v5.4.8.asciidoc[] +include::jdbc-v5.4.7.asciidoc[] include::jdbc-v5.4.6.asciidoc[] include::jdbc-v5.4.5.asciidoc[] include::jdbc-v5.4.4.asciidoc[] diff --git a/docs/versioned-plugins/integrations/jdbc-v5.4.10.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.4.10.asciidoc new file mode 100644 index 000000000..9e904fdca --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.4.10.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.10 +:release_date: 2024-05-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.10/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.4.11.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.4.11.asciidoc new file mode 100644 index 000000000..2d3890ace --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.4.11.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.11 +:release_date: 2024-05-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.11/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.4.7.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.4.7.asciidoc new file mode 100644 index 000000000..fe4d1d187 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.4.7.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.7 +:release_date: 2024-01-31 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.4.8.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.4.8.asciidoc new file mode 100644 index 000000000..684ea6822 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.4.8.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.8 +:release_date: 2024-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.4.9.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.4.9.asciidoc new file mode 100644 index 000000000..fc31b5857 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.4.9.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.4.9 +:release_date: 2024-03-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.4.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.5.0.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.5.0.asciidoc new file mode 100644 index 000000000..fa8229a53 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.5.0.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.0 +:release_date: 2024-08-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.5.1.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.5.1.asciidoc new file mode 100644 index 000000000..eb9b0b949 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.5.1.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.1 +:release_date: 2024-12-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.5.2.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.5.2.asciidoc new file mode 100644 index 000000000..1b36a567c --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.5.2.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.2 +:release_date: 2024-12-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.5.3.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.5.3.asciidoc new file mode 100644 index 000000000..75cd64ff9 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.5.3.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.5.3 +:release_date: 2025-03-07 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.5.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/jdbc-v5.6.0.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.6.0.asciidoc new file mode 100644 index 000000000..09b881ce3 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.6.0.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.0 +:release_date: 2025-05-30 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/kafka-index.asciidoc b/docs/versioned-plugins/integrations/kafka-index.asciidoc index c0bacfec8..c4ede2b34 100644 --- a/docs/versioned-plugins/integrations/kafka-index.asciidoc +++ b/docs/versioned-plugins/integrations/kafka-index.asciidoc @@ -5,6 +5,21 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-08-28 +| <> | 2025-06-12 +| <> | 2025-05-21 +| <> | 2025-05-08 +| <> | 2025-01-07 +| <> | 2024-12-18 +| <> | 2024-11-25 +| <> | 2024-10-04 +| <> | 2024-07-26 +| <> | 2024-07-02 +| <> | 2024-05-17 +| <> | 2024-04-10 +| <> | 2024-04-09 +| <> | 2024-02-06 +| <> | 2024-01-09 | <> | 2023-10-16 | <> | 2023-10-04 | <> | 2023-07-04 @@ -12,6 +27,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-02-14 | <> | 2023-01-25 | <> | 2022-12-16 +| <> | 2025-02-12 | <> | 2023-10-16 | <> | 2022-05-09 | <> | 2022-04-25 @@ -41,6 +57,21 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2019-10-15 |======================================================================= +include::kafka-v11.6.4.asciidoc[] +include::kafka-v11.6.3.asciidoc[] +include::kafka-v11.6.2.asciidoc[] +include::kafka-v11.6.1.asciidoc[] +include::kafka-v11.6.0.asciidoc[] +include::kafka-v11.5.4.asciidoc[] +include::kafka-v11.5.3.asciidoc[] +include::kafka-v11.5.2.asciidoc[] +include::kafka-v11.5.1.asciidoc[] +include::kafka-v11.5.0.asciidoc[] +include::kafka-v11.4.2.asciidoc[] +include::kafka-v11.4.1.asciidoc[] +include::kafka-v11.4.0.asciidoc[] +include::kafka-v11.3.4.asciidoc[] +include::kafka-v11.3.3.asciidoc[] include::kafka-v11.3.2.asciidoc[] include::kafka-v11.3.1.asciidoc[] include::kafka-v11.3.0.asciidoc[] @@ -48,6 +79,7 @@ include::kafka-v11.2.1.asciidoc[] include::kafka-v11.2.0.asciidoc[] include::kafka-v11.1.0.asciidoc[] include::kafka-v11.0.0.asciidoc[] +include::kafka-v10.12.2.asciidoc[] include::kafka-v10.12.1.asciidoc[] include::kafka-v10.12.0.asciidoc[] include::kafka-v10.11.0.asciidoc[] diff --git a/docs/versioned-plugins/integrations/kafka-v10.12.2.asciidoc b/docs/versioned-plugins/integrations/kafka-v10.12.2.asciidoc new file mode 100644 index 000000000..652210862 --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v10.12.2.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 2.8.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v10.12.2 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v10.12.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.3.3.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.3.3.asciidoc new file mode 100644 index 000000000..dea7d887f --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.3.3.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.3.3 +:release_date: 2024-01-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.3.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.3.4.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.3.4.asciidoc new file mode 100644 index 000000000..03df28f9f --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.3.4.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.3.4 +:release_date: 2024-02-06 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.3.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.4.0.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.4.0.asciidoc new file mode 100644 index 000000000..1cdc2f1fc --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.4.0.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.0 +:release_date: 2024-04-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.4.1.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.4.1.asciidoc new file mode 100644 index 000000000..30fbcd87c --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.4.1.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.1 +:release_date: 2024-04-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.4.2.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.4.2.asciidoc new file mode 100644 index 000000000..8869d0cbb --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.4.2.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.2 +:release_date: 2024-05-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.5.0.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.5.0.asciidoc new file mode 100644 index 000000000..f42c0d7a6 --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.5.0.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.0 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.5.1.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.5.1.asciidoc new file mode 100644 index 000000000..480a9f1fd --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.5.1.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.1 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.5.2.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.5.2.asciidoc new file mode 100644 index 000000000..0065398eb --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.5.2.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.4.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.2 +:release_date: 2024-10-04 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.5.3.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.5.3.asciidoc new file mode 100644 index 000000000..ce543a00c --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.5.3.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.7.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.3 +:release_date: 2024-11-25 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.5.4.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.5.4.asciidoc new file mode 100644 index 000000000..7542866ff --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.5.4.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.8.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.4 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.6.0.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.6.0.asciidoc new file mode 100644 index 000000000..c949c5ba4 --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.6.0.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.8.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.0 +:release_date: 2025-01-07 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.6.1.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.6.1.asciidoc new file mode 100644 index 000000000..e259b8bbe --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.6.1.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.8.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.1 +:release_date: 2025-05-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.6.2.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.6.2.asciidoc new file mode 100644 index 000000000..543822ef5 --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.6.2.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.8.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.2 +:release_date: 2025-05-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.6.3.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.6.3.asciidoc new file mode 100644 index 000000000..1bf27bead --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.6.3.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.9.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.3 +:release_date: 2025-06-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/kafka-v11.6.4.asciidoc b/docs/versioned-plugins/integrations/kafka-v11.6.4.asciidoc new file mode 100644 index 000000000..151741b3b --- /dev/null +++ b/docs/versioned-plugins/integrations/kafka-v11.6.4.asciidoc @@ -0,0 +1,36 @@ +:plugin: kafka +:type: integration +:no_codec: +:kafka_client: 3.9.1 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.4 +:release_date: 2025-08-28 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Kafka Integration Plugin provides integrated plugins for working with the +https://kafka.apache.org/[Kafka] distributed streaming platform. + + - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin] + - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin] + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/logstash-index.asciidoc b/docs/versioned-plugins/integrations/logstash-index.asciidoc index 22c5f6464..48f0e48fb 100644 --- a/docs/versioned-plugins/integrations/logstash-index.asciidoc +++ b/docs/versioned-plugins/integrations/logstash-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-12-10 +| <> | 2024-09-12 +| <> | 2024-01-23 | <> | 2023-11-16 | <> | 2023-11-06 | <> | 2023-10-03 @@ -14,6 +17,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-09-25 |======================================================================= +include::logstash-v1.0.4.asciidoc[] +include::logstash-v1.0.3.asciidoc[] +include::logstash-v1.0.2.asciidoc[] include::logstash-v1.0.1.asciidoc[] include::logstash-v1.0.0.asciidoc[] include::logstash-v0.0.5.asciidoc[] diff --git a/docs/versioned-plugins/integrations/logstash-v1.0.2.asciidoc b/docs/versioned-plugins/integrations/logstash-v1.0.2.asciidoc new file mode 100644 index 000000000..43cec0755 --- /dev/null +++ b/docs/versioned-plugins/integrations/logstash-v1.0.2.asciidoc @@ -0,0 +1,75 @@ +:plugin: logstash +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.2 +:release_date: 2024-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another instance(s): + +* {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin] +* {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] + +[id="{version}-plugins-{type}s-{plugin}-concepts"] +===== High-level concepts + +You can configure a `logstash` output to send events to one or more `logstash` inputs, which are each in another pipeline that is running in different processes or on a different host. + +To do so, you should first configure the downstream pipeline with a `logstash` input plugin, bound to an available port so that it can listen for inbound connections. +Security is enabled by default, so you will need to either provide identity material or disable SSL. + +NOTE: You will need a TCP route from the upstream pipeline to the interface that the downstream pipeline is bound to. + +[source,shell] +---- +input { + logstash { + port => 9800 + + # SSL IDENTITY <1> + ssl_keystore_path => "/path/to/identity.p12" + ssl_keystore_password => "${SSL_IDENTITY_PASSWORD}" + } +} +---- +<1> Identity material typically should include identity claims about the hostnames and ip addresses that will be used by upstream output plugins. + +Once the downstream pipeline is configured and running, you may send events from any number of upstream pipelines by adding a `logstash` output plugin that points to the downstream input. +You may need to configure SSL to trust the certificates presented by the downstream input plugin. + +[source,shell] +---- +output { + logstash { + hosts => ["10.0.0.123:9800", "10.0.0.125:9801"] + + # SSL TRUST <1> + ssl_truststore_path => "/path/to/truststore.p12" + ssl_truststore_password => "${SSL_TRUST_PASSWORD}" + } +} +---- +<1> Unless SSL is disabled or the downstream input is expected to present certificates signed by globally-trusted authorities, you will likely need to provide a source-of-trust. + +[id="{version}-plugins-{type}s-{plugin}-load-balancing"] +==== Load Balancing + +When a `logstash` output is configured to send to multiple `hosts`, it distributes events in batches to _all_ of those downstream hosts fairly, favoring those without recent errors. This increases the likelihood of each batch being routed to a downstream that is up and has capacity to receive events. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/logstash-v1.0.3.asciidoc b/docs/versioned-plugins/integrations/logstash-v1.0.3.asciidoc new file mode 100644 index 000000000..4ba70b062 --- /dev/null +++ b/docs/versioned-plugins/integrations/logstash-v1.0.3.asciidoc @@ -0,0 +1,75 @@ +:plugin: logstash +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.3 +:release_date: 2024-09-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another instance(s): + +* {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin] +* {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] + +[id="{version}-plugins-{type}s-{plugin}-concepts"] +===== High-level concepts + +You can configure a `logstash` output to send events to one or more `logstash` inputs, which are each in another pipeline that is running in different processes or on a different host. + +To do so, you should first configure the downstream pipeline with a `logstash` input plugin, bound to an available port so that it can listen for inbound connections. +Security is enabled by default, so you will need to either provide identity material or disable SSL. + +NOTE: You will need a TCP route from the upstream pipeline to the interface that the downstream pipeline is bound to. + +[source,shell] +---- +input { + logstash { + port => 9800 + + # SSL IDENTITY <1> + ssl_keystore_path => "/path/to/identity.p12" + ssl_keystore_password => "${SSL_IDENTITY_PASSWORD}" + } +} +---- +<1> Identity material typically should include identity claims about the hostnames and ip addresses that will be used by upstream output plugins. + +Once the downstream pipeline is configured and running, you may send events from any number of upstream pipelines by adding a `logstash` output plugin that points to the downstream input. +You may need to configure SSL to trust the certificates presented by the downstream input plugin. + +[source,shell] +---- +output { + logstash { + hosts => ["10.0.0.123:9800", "10.0.0.125:9801"] + + # SSL TRUST <1> + ssl_truststore_path => "/path/to/truststore.p12" + ssl_truststore_password => "${SSL_TRUST_PASSWORD}" + } +} +---- +<1> Unless SSL is disabled or the downstream input is expected to present certificates signed by globally-trusted authorities, you will likely need to provide a source-of-trust. + +[id="{version}-plugins-{type}s-{plugin}-load-balancing"] +==== Load Balancing + +When a `logstash` output is configured to send to multiple `hosts`, it distributes events in batches to _all_ of those downstream hosts fairly, favoring those without recent errors. This increases the likelihood of each batch being routed to a downstream that is up and has capacity to receive events. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/logstash-v1.0.4.asciidoc b/docs/versioned-plugins/integrations/logstash-v1.0.4.asciidoc new file mode 100644 index 000000000..63d64b77f --- /dev/null +++ b/docs/versioned-plugins/integrations/logstash-v1.0.4.asciidoc @@ -0,0 +1,75 @@ +:plugin: logstash +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.4 +:release_date: 2024-12-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another instance(s): + +* {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin] +* {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] + +[id="{version}-plugins-{type}s-{plugin}-concepts"] +===== High-level concepts + +You can configure a `logstash` output to send events to one or more `logstash` inputs, which are each in another pipeline that is running in different processes or on a different host. + +To do so, you should first configure the downstream pipeline with a `logstash` input plugin, bound to an available port so that it can listen for inbound connections. +Security is enabled by default, so you will need to either provide identity material or disable SSL. + +NOTE: You will need a TCP route from the upstream pipeline to the interface that the downstream pipeline is bound to. + +[source,shell] +---- +input { + logstash { + port => 9800 + + # SSL IDENTITY <1> + ssl_keystore_path => "/path/to/identity.p12" + ssl_keystore_password => "${SSL_IDENTITY_PASSWORD}" + } +} +---- +<1> Identity material typically should include identity claims about the hostnames and ip addresses that will be used by upstream output plugins. + +Once the downstream pipeline is configured and running, you may send events from any number of upstream pipelines by adding a `logstash` output plugin that points to the downstream input. +You may need to configure SSL to trust the certificates presented by the downstream input plugin. + +[source,shell] +---- +output { + logstash { + hosts => ["10.0.0.123:9800", "10.0.0.125:9801"] + + # SSL TRUST <1> + ssl_truststore_path => "/path/to/truststore.p12" + ssl_truststore_password => "${SSL_TRUST_PASSWORD}" + } +} +---- +<1> Unless SSL is disabled or the downstream input is expected to present certificates signed by globally-trusted authorities, you will likely need to provide a source-of-trust. + +[id="{version}-plugins-{type}s-{plugin}-load-balancing"] +==== Load Balancing + +When a `logstash` output is configured to send to multiple `hosts`, it distributes events in batches to _all_ of those downstream hosts fairly, favoring those without recent errors. This increases the likelihood of each batch being routed to a downstream that is up and has capacity to receive events. + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/rabbitmq-index.asciidoc b/docs/versioned-plugins/integrations/rabbitmq-index.asciidoc index d7381d77a..c6177255a 100644 --- a/docs/versioned-plugins/integrations/rabbitmq-index.asciidoc +++ b/docs/versioned-plugins/integrations/rabbitmq-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-09-16 | <> | 2023-06-09 | <> | 2023-05-30 | <> | 2022-10-04 @@ -18,6 +19,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2019-10-15 |======================================================================= +include::rabbitmq-v7.4.0.asciidoc[] include::rabbitmq-v7.3.3.asciidoc[] include::rabbitmq-v7.3.2.asciidoc[] include::rabbitmq-v7.3.1.asciidoc[] diff --git a/docs/versioned-plugins/integrations/rabbitmq-v7.4.0.asciidoc b/docs/versioned-plugins/integrations/rabbitmq-v7.4.0.asciidoc new file mode 100644 index 000000000..53d394a81 --- /dev/null +++ b/docs/versioned-plugins/integrations/rabbitmq-v7.4.0.asciidoc @@ -0,0 +1,29 @@ +:plugin: rabbitmq +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.4.0 +:release_date: 2024-09-16 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-rabbitmq/blob/v7.4.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Rabbitmq Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The RabbitMQ Integration Plugin provides integrated plugins for working with http://www.rabbitmq.com/[RabbitMQ]. + + - {logstash-ref}/plugins-inputs-rabbitmq.html[RabbitMQ Input Plugin] + - {logstash-ref}/plugins-outputs-rabbitmq.html[RabbitMQ Output Plugin] + +:no_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/integrations/snmp-index.asciidoc b/docs/versioned-plugins/integrations/snmp-index.asciidoc new file mode 100644 index 000000000..550d18c32 --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-index.asciidoc @@ -0,0 +1,28 @@ +:plugin: snmp +:type: integration + +include::{include_path}/version-list-intro.asciidoc[] + +|======================================================================= +| Version | Release Date +| <> | 2025-08-18 +| <> | 2025-07-17 +| <> | 2025-01-23 +| <> | 2025-01-06 +| <> | 2024-07-08 +| <> | 2024-06-11 +| <> | 2024-05-20 +| <> | 2024-05-17 +| <> | 2024-05-03 +|======================================================================= + +include::snmp-v4.1.0.asciidoc[] +include::snmp-v4.0.7.asciidoc[] +include::snmp-v4.0.6.asciidoc[] +include::snmp-v4.0.5.asciidoc[] +include::snmp-v4.0.4.asciidoc[] +include::snmp-v4.0.3.asciidoc[] +include::snmp-v4.0.2.asciidoc[] +include::snmp-v4.0.1.asciidoc[] +include::snmp-v4.0.0.asciidoc[] + diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.0.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.0.asciidoc new file mode 100644 index 000000000..425ece862 --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.0.asciidoc @@ -0,0 +1,202 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.0 +:release_date: 2024-05-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +// include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The SNMP integration plugin includes: + +* SNMP input plugin +* SNMPtrap input plugin + +// Uncomment and replace with above bullets after generated the initial versions of the target files +// - {logstash-ref}/plugins-input-snmp.html[SNMP input plugin] +// - {logstash-ref}/plugins-input-snmptrap.html[Snmptrap input plugin] + +The new the `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource managenment, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[IMPORTANT] +-- +Uninstall the `logstash-input-snmp` and `logstash-input-snmptrap` plugins _before_ you install the `logstash-integration-snmp` plugin. + +* `bin/logstash-plugin remove logstash-input-snmp` + +* `bin/logstash-plugin remove logstash-input-snmptrap` +-- + +// ToDo: Add guidance and set expectations for when snmp integration is installed by default + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +:no_codec!: + + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: + + diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.1.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.1.asciidoc new file mode 100644 index 000000000..5445b57cf --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.1.asciidoc @@ -0,0 +1,196 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.1 +:release_date: 2024-05-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of <<{version}-plugins-{type}s-{plugin}-migration,behavioral and mapping differences>> between current stand-alone plugins and the new versions included in the `integration-snmp`. +The information in this topic can help. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new the `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource managenment, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.2.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.2.asciidoc new file mode 100644 index 000000000..80a89bebe --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.2.asciidoc @@ -0,0 +1,196 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.2 +:release_date: 2024-05-20 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of <<{version}-plugins-{type}s-{plugin}-migration,behavioral and mapping differences>> between current stand-alone plugins and the new versions included in the `integration-snmp`. +The information in this topic can help. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new the `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource managenment, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.3.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.3.asciidoc new file mode 100644 index 000000000..85567d43c --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.3.asciidoc @@ -0,0 +1,196 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.3 +:release_date: 2024-06-11 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +experimental[] + +.Technical Preview +**** +The new `integration-snmp` plugin, and its component plugins--`input-snmp` and `input-snmptrap`--are available in _Technical Preview_ and can be installed on the latest Logstash 7.x and 8.x versions. + +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the snmp input plugin contained in this integration. +(If you want to opt into the Technical Preview for the `integration-snmp` plugin, run `bin/logstash-plugin install logstash-integration-snmp`.) + +Be aware of <<{version}-plugins-{type}s-{plugin}-migration,behavioral and mapping differences>> between current stand-alone plugins and the new versions included in the `integration-snmp`. +The information in this topic can help. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new the `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource managenment, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.4.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.4.asciidoc new file mode 100644 index 000000000..41deecae7 --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.4.asciidoc @@ -0,0 +1,193 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.4 +:release_date: 2024-07-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Announcing the new SNMP integration plugin +**** +The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default. +This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later. +Earlier versions of the stand-alone plugins that were bundled with {ls} by default will be replaced by the 4.0.0+ version contained in this new integration. + +IMPORTANT: Before you upgrade to {ls} 8.15.0 that includes this new integration by default, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new the `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource managenment, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.5.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.5.asciidoc new file mode 100644 index 000000000..98523f30c --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.5.asciidoc @@ -0,0 +1,193 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.5 +:release_date: 2025-01-06 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Announcing the new SNMP integration plugin +**** +The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default. +This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later. +Earlier versions of the stand-alone plugins that were bundled with {ls} by default will be replaced by the 4.0.0+ version contained in this new integration. + +IMPORTANT: Before you upgrade to {ls} 8.15.0 that includes this new integration by default, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.6.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.6.asciidoc new file mode 100644 index 000000000..4e627c4a1 --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.6.asciidoc @@ -0,0 +1,193 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.6 +:release_date: 2025-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Announcing the new SNMP integration plugin +**** +The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default. +This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later. +Earlier versions of the stand-alone plugins that were bundled with {ls} by default will be replaced by the 4.0.0+ version contained in this new integration. + +IMPORTANT: Before you upgrade to {ls} 8.15.0 that includes this new integration by default, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmptrap[logstash-input-snmptrap] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/snmp-v4.0.7.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.0.7.asciidoc new file mode 100644 index 000000000..d4b286443 --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.0.7.asciidoc @@ -0,0 +1,193 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.0.7 +:release_date: 2025-07-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.0.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Announcing the new SNMP integration plugin +**** +The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default. +This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later. +Earlier versions of the stand-alone plugins that were bundled with {ls} by default will be replaced by the 4.0.0+ version contained in this new integration. + +IMPORTANT: Before you upgrade to {ls} 8.15.0 that includes this new integration by default, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmptrap[logstash-input-snmptrap] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/integrations/snmp-v4.1.0.asciidoc b/docs/versioned-plugins/integrations/snmp-v4.1.0.asciidoc new file mode 100644 index 000000000..06e81225a --- /dev/null +++ b/docs/versioned-plugins/integrations/snmp-v4.1.0.asciidoc @@ -0,0 +1,193 @@ +:plugin: snmp +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.1.0 +:release_date: 2025-08-18 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-snmp/blob/v4.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== SNMP Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +.Announcing the new SNMP integration plugin +**** +The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default. +This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later. +Earlier versions of the stand-alone plugins that were bundled with {ls} by default will be replaced by the 4.0.0+ version contained in this new integration. + +IMPORTANT: Before you upgrade to {ls} 8.15.0 that includes this new integration by default, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. +**** + +==== Description + +The SNMP integration plugin includes: + +* link:{logstash-ref}/plugins-inputs-snmp.html[SNMP input plugin] +* link:{logstash-ref}/plugins-inputs-snmptrap.html[Snmptrap input plugin] + +The new `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +In this section, we'll cover: + +* <<{version}-plugins-{type}s-{plugin}-migration>> +* <<{version}-plugins-{type}s-{plugin}-import-mibs>> + +[id="{version}-plugins-{type}s-{plugin}-migration"] +==== Migrating to `logstash-integration-snmp` from individual plugins + +You'll retain and expand the functionality of existing stand-alone plugins, but in a more compact, integrated package. +In this section, we'll note mapping and behavioral changes, and explain how to preserve current behavior if needed. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmp"] +===== Migration notes: `logstash-input-snmp` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmp` plugin offers the same +capabilities as the stand-alone https://github.com/logstash-plugins/logstash-input-snmp[logstash-input-snmp]. + +You might need to address some behavior changes depending on the use-case and how the ingested data is being handled through the pipeline. + + +[id="{version}-plugins-{type}s-{plugin}-input-snmp-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmp` + +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* An *unknown variable type* falls back to the `string` representation instead of logging an error as it did in with the stand-alone `logstash-input-snmp`. +This change should not affect existing pipelines, unless they have custom error handlers that rely on specific error messages. + +[id="{version}-plugins-{type}s-{plugin}-migration-input-snmptrap"] +===== Migration notes: `logstash-input-snmptrap` + +As a component of the new `logstash-integration-snmp` plugin, the `logstash-input-snmptrap` plugin offers _almost the same +capabilities_ as the stand-alone https://github.com/logstash-plugins/logstash-input-snmptrap[logstash-input-snmptrap] plugin. + +You might need to address some behavior changes depending on your use case and how the ingested data is being handled through the pipeline. + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-mapping"] +===== Changes to mapping and error logging: `logstash-input-snmptrap` + +* The *PDU variable bindings* are mapped into the {ls} event using the defined data type. + By default, the stand-alone `logstash-input-snmptrap` plugin converts all of the data to `string`, ignoring the original type. + If this behavior is not what you want, you can use a filter to retain the original type. +* *SNMP `TimeTicks` variables* are mapped as `Long` timestamps instead of formatted date string (`%d days, %02d:%02d:%02d.%02d`). +* *`null` variables values* are mapped using the string `null` instead of `Null` (upper-case N). +* *No such instance errors* are mapped as `error: no such instance currently exists at this OID string` instead of `noSuchInstance`. +* *No such object errors* are mapped as `error: no such object currently exists at this OID string` instead of `noSuchObject`. +* *End of MIB view errors* are mapped as `error: end of MIB view` instead of `endOfMibView`. +* The previous generation (stand-alone) input-snmptrap plugin formatted the *`message` field* as +a ruby-snmp `SNMP::SNMPv1_Trap` object representation. ++ +[source,sh] +---- +], @timestamp=#, @generic_trap=6, @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#, @specific_trap=99> +---- ++ +The new integrated `input-snmptrap` plugin uses JSON to format *`message` field*. ++ +[source,json] +---- +{"error_index":0, "variable_bindings":{"1.3.6.1.6.3.1.1.4.1.0":"SNMPv2-MIB::coldStart", "1.3.6.1.2.1.1.3.0":0}, "error_status":0, "type":"TRAP", "error_status_text":"Success", "community":"public", "version":"2c", "request_id":1436216872} +---- + +// ToDo: Add more details wrt PDU variable binding. Which filter? Add sample config? + +[id="{version}-plugins-{type}s-{plugin}-input-snmptrap-compat"] +===== Maintain maximum compatibility with previous implementation + +If needed, you can configure the new `logstash-integration-snmp` plugin to maintain maximum compatibility with the previous (stand-alone) +version of the https://github.com/logstash-plugins/logstash-input-snmp[input-snmp] plugin. + +[source,ruby] +---- +input { + snmptrap { + use_provided_mibs => false + oid_mapping_format => 'ruby_snmp' + oid_map_field_values => true + } +} +---- + +// ToDo: Any considerations that we should point out? + +[id="{version}-plugins-{type}s-{plugin}-import-mibs"] +==== Importing MIBs + +The SNMP plugins already include the IETF MIBs (management information bases) and these do not need to be imported. +To disable the bundled MIBs set the `use_provided_mibs` option to `false`. + +Any other MIB will need to be manually imported to provide mapping of the numeric OIDs to MIB field names in the resulting event. + +To import a MIB, the OSS https://www.ibr.cs.tu-bs.de/projects/libsmi/[libsmi library] is required. +libsmi is available and installable on most operating systems. + +To import a MIB, you need to first convert the ASN.1 MIB file into a `.dic` file using the libsmi `smidump` command line utility. + +*Example (using `RFC1213-MIB` file)* + +[source,sh] +----- +$ smidump --level=1 -k -f python RFC1213-MIB > RFC1213-MIB.dic +----- + +Note that the resulting file as output by `smidump` must have the `.dic` extension. + +[id="{version}-plugins-{type}s-{plugin}-locate-mibs"] +===== Preventing a `failed to locate MIB module` error + +The `smidump` function looks for MIB dependencies in its pre-configured paths list. +To avoid the `failed to locate MIB module` error, you may need to provide the MIBs locations in your particular environment. + +The recommended ways to provide the additional path configuration are: + +* an environment variable, or +* a config file to provide the additional path configuration. + +See the "MODULE LOCATIONS" section of the https://www.ibr.cs.tu-bs.de/projects/libsmi/smi_config.html#MODULE%20LOCATIONS[smi_config documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-env-var"] +===== Option 1: Use an environment variable + +Set the `SMIPATH` env var with the path to your MIBs. +Be sure to include a prepended colon (`:`) for the path. + +[source,sh] +----- +$ SMIPATH=":/path/to/mibs/" smidump -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic <1> +----- +<1> Notice the colon that precedes the path definition. + +[id="{version}-plugins-{type}s-{plugin}-mib-config"] +===== Option 2: Provide a configuration file + +The other approach is to create a configuration file with the `path` option. For example, you could create a file called `smi.conf`: + +[source,sh] +----- +path :/path/to/mibs/ +----- + +And use the config with smidump: + +[source,sh] +----- +$ smidump -c smi.conf -k -f python CISCO-PROCESS-MIB.mib > CISCO-PROCESS-MIB_my.dic +----- + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/cloudwatch-index.asciidoc b/docs/versioned-plugins/outputs/cloudwatch-index.asciidoc index 762257ea3..5f5706c62 100644 --- a/docs/versioned-plugins/outputs/cloudwatch-index.asciidoc +++ b/docs/versioned-plugins/outputs/cloudwatch-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -23,6 +27,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::cloudwatch-v7.2.1.asciidoc[] +include::cloudwatch-v7.2.0.asciidoc[] +include::cloudwatch-v7.1.8.asciidoc[] +include::cloudwatch-v7.1.7.asciidoc[] include::cloudwatch-v7.1.6.asciidoc[] include::cloudwatch-v7.1.5.asciidoc[] include::cloudwatch-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/outputs/cloudwatch-v7.1.7.asciidoc b/docs/versioned-plugins/outputs/cloudwatch-v7.1.7.asciidoc new file mode 100644 index 000000000..8ac218aa0 --- /dev/null +++ b/docs/versioned-plugins/outputs/cloudwatch-v7.1.7.asciidoc @@ -0,0 +1,332 @@ +:integration: aws +:plugin: cloudwatch +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This output lets you aggregate and send metric data to AWS CloudWatch + +==== Summary: +This plugin is intended to be used on a logstash indexer agent (but that +is not the only way, see below.) In the intended scenario, one cloudwatch +output plugin is configured, on the logstash indexer node, with just AWS API +credentials, and possibly a region and/or a namespace. The output looks +for fields present in events, and when it finds them, it uses them to +calculate aggregate statistics. If the `metricname` option is set in this +output, then any events which pass through it will be aggregated & sent to +CloudWatch, but that is not recommended. The intended use is to NOT set the +metricname option here, and instead to add a `CW_metricname` field (and other +fields) to only the events you want sent to CloudWatch. + +When events pass through this output they are queued for background +aggregation and sending, which happens every minute by default. The +queue has a maximum size, and when it is full aggregated statistics will be +sent to CloudWatch ahead of schedule. Whenever this happens a warning +message is written to logstash's log. If you see this you should increase +the `queue_size` configuration option to avoid the extra API calls. The queue +is emptied every time we send data to CloudWatch. + +Note: when logstash is stopped the queue is destroyed before it can be processed. +This is a known limitation of logstash and will hopefully be addressed in a +future version. + +==== Details: +There are two ways to configure this plugin, and they can be used in +combination: event fields & per-output defaults + +Event Field configuration... +You add fields to your events in inputs & filters and this output reads +those fields to aggregate events. The names of the fields read are +configurable via the `field_*` options. + +Per-output defaults... +You set universal defaults in this output plugin's configuration, and +if an event does not have a field for that option then the default is +used. + +Notice, the event fields take precedence over the per-output defaults. + +At a minimum events must have a "metric name" to be sent to CloudWatch. +This can be achieved either by providing a default here OR by adding a +`CW_metricname` field. By default, if no other configuration is provided +besides a metric name, then events will be counted (Unit: Count, Value: 1) +by their metric name (either a default or from their `CW_metricname` field) + +Other fields which can be added to events to modify the behavior of this +plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and +`CW_dimensions`. All of these field names are configurable in +this output. You can also set per-output defaults for any of them. +See below for details. + +Read more about http://aws.amazon.com/cloudwatch/[AWS CloudWatch], +and the specific of API endpoint this output uses, +http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html[PutMetricData] + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dimensions>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-field_dimensions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_unit>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-timeframe>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-unit>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", "Gigabytes", "Terabytes", "Bits", "Kilobits", "Megabits", "Gigabits", "Terabits", "Percent", "Count", "Bytes/Second", "Kilobytes/Second", "Megabytes/Second", "Gigabytes/Second", "Terabytes/Second", "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +How many data points can be given in one call to the CloudWatch API + +[id="{version}-plugins-{type}s-{plugin}-dimensions"] +===== `dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field + +[id="{version}-plugins-{type}s-{plugin}-field_dimensions"] +===== `field_dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_dimensions"` + +The name of the field used to set the dimensions on an event metric +The field named here, if present in an event, must have an array of +one or more key & value pairs, for example... + `add_field => [ "CW_dimensions", "Environment", "CW_dimensions", "prod" ]` +or, equivalently... + `add_field => [ "CW_dimensions", "Environment" ]` + `add_field => [ "CW_dimensions", "prod" ]` + +[id="{version}-plugins-{type}s-{plugin}-field_metricname"] +===== `field_metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_metricname"` + +The name of the field used to set the metric name on an event +The author of this plugin recommends adding this field to events in inputs & +filters rather than using the per-output default setting so that one output +plugin on your logstash indexer can serve all events (which of course had +fields set on your logstash shippers.) + +[id="{version}-plugins-{type}s-{plugin}-field_namespace"] +===== `field_namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_namespace"` + +The name of the field used to set a different namespace per event +Note: Only one namespace can be sent to CloudWatch per API call +so setting different namespaces will increase the number of API calls +and those cost money. + +[id="{version}-plugins-{type}s-{plugin}-field_unit"] +===== `field_unit` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_unit"` + +The name of the field used to set the unit on an event metric + +[id="{version}-plugins-{type}s-{plugin}-field_value"] +===== `field_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_value"` + +The name of the field used to set the value (float) on an event metric + +[id="{version}-plugins-{type}s-{plugin}-metricname"] +===== `metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The default metric name to use for events which do not have a `CW_metricname` field. +Beware: If this is provided then all events which pass through this output will be aggregated and +sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you +will probably want to also restrict events from passing through this output using event +type, tag, and field matching + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Logstash"` + +The default namespace to use for events which do not have a `CW_namespace` field + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue_size"] +===== `queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule +Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-timeframe"] +===== `timeframe` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +Constants +aggregate_key members +Units +How often to send data to CloudWatch +This does not affect the event timestamps, events will always have their +actual timestamp (to-the-minute) sent to CloudWatch. + +We only call the API if there is data to send. + +See the Rufus Scheduler docs for an https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler[explanation of allowed values] + +[id="{version}-plugins-{type}s-{plugin}-unit"] +===== `unit` + + * Value can be any of: `Seconds`, `Microseconds`, `Milliseconds`, `Bytes`, `Kilobytes`, `Megabytes`, `Gigabytes`, `Terabytes`, `Bits`, `Kilobits`, `Megabits`, `Gigabits`, `Terabits`, `Percent`, `Count`, `Bytes/Second`, `Kilobytes/Second`, `Megabytes/Second`, `Gigabytes/Second`, `Terabytes/Second`, `Bits/Second`, `Kilobits/Second`, `Megabits/Second`, `Gigabits/Second`, `Terabits/Second`, `Count/Second`, `None` + * Default value is `"Count"` + +The default unit to use for events which do not have a `CW_unit` field +If you set this option you should probably set the "value" option along with it + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-value"] +===== `value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1"` + +The default value to use for events which do not have a `CW_value` field +If provided, this must be a string which can be converted to a float, for example... + "1", "2.34", ".5", and "0.67" +If you set this option you should probably set the `unit` option along with it + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/cloudwatch-v7.1.8.asciidoc b/docs/versioned-plugins/outputs/cloudwatch-v7.1.8.asciidoc new file mode 100644 index 000000000..4ad04b121 --- /dev/null +++ b/docs/versioned-plugins/outputs/cloudwatch-v7.1.8.asciidoc @@ -0,0 +1,332 @@ +:integration: aws +:plugin: cloudwatch +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This output lets you aggregate and send metric data to AWS CloudWatch + +==== Summary: +This plugin is intended to be used on a logstash indexer agent (but that +is not the only way, see below.) In the intended scenario, one cloudwatch +output plugin is configured, on the logstash indexer node, with just AWS API +credentials, and possibly a region and/or a namespace. The output looks +for fields present in events, and when it finds them, it uses them to +calculate aggregate statistics. If the `metricname` option is set in this +output, then any events which pass through it will be aggregated & sent to +CloudWatch, but that is not recommended. The intended use is to NOT set the +metricname option here, and instead to add a `CW_metricname` field (and other +fields) to only the events you want sent to CloudWatch. + +When events pass through this output they are queued for background +aggregation and sending, which happens every minute by default. The +queue has a maximum size, and when it is full aggregated statistics will be +sent to CloudWatch ahead of schedule. Whenever this happens a warning +message is written to logstash's log. If you see this you should increase +the `queue_size` configuration option to avoid the extra API calls. The queue +is emptied every time we send data to CloudWatch. + +Note: when logstash is stopped the queue is destroyed before it can be processed. +This is a known limitation of logstash and will hopefully be addressed in a +future version. + +==== Details: +There are two ways to configure this plugin, and they can be used in +combination: event fields & per-output defaults + +Event Field configuration... +You add fields to your events in inputs & filters and this output reads +those fields to aggregate events. The names of the fields read are +configurable via the `field_*` options. + +Per-output defaults... +You set universal defaults in this output plugin's configuration, and +if an event does not have a field for that option then the default is +used. + +Notice, the event fields take precedence over the per-output defaults. + +At a minimum events must have a "metric name" to be sent to CloudWatch. +This can be achieved either by providing a default here OR by adding a +`CW_metricname` field. By default, if no other configuration is provided +besides a metric name, then events will be counted (Unit: Count, Value: 1) +by their metric name (either a default or from their `CW_metricname` field) + +Other fields which can be added to events to modify the behavior of this +plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and +`CW_dimensions`. All of these field names are configurable in +this output. You can also set per-output defaults for any of them. +See below for details. + +Read more about http://aws.amazon.com/cloudwatch/[AWS CloudWatch], +and the specific of API endpoint this output uses, +http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html[PutMetricData] + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dimensions>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-field_dimensions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_unit>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-timeframe>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-unit>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", "Gigabytes", "Terabytes", "Bits", "Kilobits", "Megabits", "Gigabits", "Terabits", "Percent", "Count", "Bytes/Second", "Kilobytes/Second", "Megabytes/Second", "Gigabytes/Second", "Terabytes/Second", "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +How many data points can be given in one call to the CloudWatch API + +[id="{version}-plugins-{type}s-{plugin}-dimensions"] +===== `dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field + +[id="{version}-plugins-{type}s-{plugin}-field_dimensions"] +===== `field_dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_dimensions"` + +The name of the field used to set the dimensions on an event metric +The field named here, if present in an event, must have an array of +one or more key & value pairs, for example... + `add_field => [ "CW_dimensions", "Environment", "CW_dimensions", "prod" ]` +or, equivalently... + `add_field => [ "CW_dimensions", "Environment" ]` + `add_field => [ "CW_dimensions", "prod" ]` + +[id="{version}-plugins-{type}s-{plugin}-field_metricname"] +===== `field_metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_metricname"` + +The name of the field used to set the metric name on an event +The author of this plugin recommends adding this field to events in inputs & +filters rather than using the per-output default setting so that one output +plugin on your logstash indexer can serve all events (which of course had +fields set on your logstash shippers.) + +[id="{version}-plugins-{type}s-{plugin}-field_namespace"] +===== `field_namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_namespace"` + +The name of the field used to set a different namespace per event +Note: Only one namespace can be sent to CloudWatch per API call +so setting different namespaces will increase the number of API calls +and those cost money. + +[id="{version}-plugins-{type}s-{plugin}-field_unit"] +===== `field_unit` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_unit"` + +The name of the field used to set the unit on an event metric + +[id="{version}-plugins-{type}s-{plugin}-field_value"] +===== `field_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_value"` + +The name of the field used to set the value (float) on an event metric + +[id="{version}-plugins-{type}s-{plugin}-metricname"] +===== `metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The default metric name to use for events which do not have a `CW_metricname` field. +Beware: If this is provided then all events which pass through this output will be aggregated and +sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you +will probably want to also restrict events from passing through this output using event +type, tag, and field matching + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Logstash"` + +The default namespace to use for events which do not have a `CW_namespace` field + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue_size"] +===== `queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule +Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-timeframe"] +===== `timeframe` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +Constants +aggregate_key members +Units +How often to send data to CloudWatch +This does not affect the event timestamps, events will always have their +actual timestamp (to-the-minute) sent to CloudWatch. + +We only call the API if there is data to send. + +See the Rufus Scheduler docs for an https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler[explanation of allowed values] + +[id="{version}-plugins-{type}s-{plugin}-unit"] +===== `unit` + + * Value can be any of: `Seconds`, `Microseconds`, `Milliseconds`, `Bytes`, `Kilobytes`, `Megabytes`, `Gigabytes`, `Terabytes`, `Bits`, `Kilobits`, `Megabits`, `Gigabits`, `Terabits`, `Percent`, `Count`, `Bytes/Second`, `Kilobytes/Second`, `Megabytes/Second`, `Gigabytes/Second`, `Terabytes/Second`, `Bits/Second`, `Kilobits/Second`, `Megabits/Second`, `Gigabits/Second`, `Terabits/Second`, `Count/Second`, `None` + * Default value is `"Count"` + +The default unit to use for events which do not have a `CW_unit` field +If you set this option you should probably set the "value" option along with it + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-value"] +===== `value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1"` + +The default value to use for events which do not have a `CW_value` field +If provided, this must be a string which can be converted to a float, for example... + "1", "2.34", ".5", and "0.67" +If you set this option you should probably set the `unit` option along with it + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/cloudwatch-v7.2.0.asciidoc b/docs/versioned-plugins/outputs/cloudwatch-v7.2.0.asciidoc new file mode 100644 index 000000000..b15c85a5b --- /dev/null +++ b/docs/versioned-plugins/outputs/cloudwatch-v7.2.0.asciidoc @@ -0,0 +1,343 @@ +:integration: aws +:plugin: cloudwatch +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This output lets you aggregate and send metric data to AWS CloudWatch + +==== Summary: +This plugin is intended to be used on a logstash indexer agent (but that +is not the only way, see below.) In the intended scenario, one cloudwatch +output plugin is configured, on the logstash indexer node, with just AWS API +credentials, and possibly a region and/or a namespace. The output looks +for fields present in events, and when it finds them, it uses them to +calculate aggregate statistics. If the `metricname` option is set in this +output, then any events which pass through it will be aggregated & sent to +CloudWatch, but that is not recommended. The intended use is to NOT set the +metricname option here, and instead to add a `CW_metricname` field (and other +fields) to only the events you want sent to CloudWatch. + +When events pass through this output they are queued for background +aggregation and sending, which happens every minute by default. The +queue has a maximum size, and when it is full aggregated statistics will be +sent to CloudWatch ahead of schedule. Whenever this happens a warning +message is written to logstash's log. If you see this you should increase +the `queue_size` configuration option to avoid the extra API calls. The queue +is emptied every time we send data to CloudWatch. + +Note: when logstash is stopped the queue is destroyed before it can be processed. +This is a known limitation of logstash and will hopefully be addressed in a +future version. + +==== Details: +There are two ways to configure this plugin, and they can be used in +combination: event fields & per-output defaults + +Event Field configuration... +You add fields to your events in inputs & filters and this output reads +those fields to aggregate events. The names of the fields read are +configurable via the `field_*` options. + +Per-output defaults... +You set universal defaults in this output plugin's configuration, and +if an event does not have a field for that option then the default is +used. + +Notice, the event fields take precedence over the per-output defaults. + +At a minimum events must have a "metric name" to be sent to CloudWatch. +This can be achieved either by providing a default here OR by adding a +`CW_metricname` field. By default, if no other configuration is provided +besides a metric name, then events will be counted (Unit: Count, Value: 1) +by their metric name (either a default or from their `CW_metricname` field) + +Other fields which can be added to events to modify the behavior of this +plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and +`CW_dimensions`. All of these field names are configurable in +this output. You can also set per-output defaults for any of them. +See below for details. + +Read more about http://aws.amazon.com/cloudwatch/[AWS CloudWatch], +and the specific of API endpoint this output uses, +http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html[PutMetricData] + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dimensions>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-field_dimensions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_unit>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-timeframe>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-unit>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", "Gigabytes", "Terabytes", "Bits", "Kilobits", "Megabits", "Gigabits", "Terabits", "Percent", "Count", "Bytes/Second", "Kilobytes/Second", "Megabytes/Second", "Gigabytes/Second", "Terabytes/Second", "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +How many data points can be given in one call to the CloudWatch API + +[id="{version}-plugins-{type}s-{plugin}-dimensions"] +===== `dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field + +[id="{version}-plugins-{type}s-{plugin}-field_dimensions"] +===== `field_dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_dimensions"` + +The name of the field used to set the dimensions on an event metric +The field named here, if present in an event, must have an array of +one or more key & value pairs, for example... + `add_field => [ "CW_dimensions", "Environment", "CW_dimensions", "prod" ]` +or, equivalently... + `add_field => [ "CW_dimensions", "Environment" ]` + `add_field => [ "CW_dimensions", "prod" ]` + +[id="{version}-plugins-{type}s-{plugin}-field_metricname"] +===== `field_metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_metricname"` + +The name of the field used to set the metric name on an event +The author of this plugin recommends adding this field to events in inputs & +filters rather than using the per-output default setting so that one output +plugin on your logstash indexer can serve all events (which of course had +fields set on your logstash shippers.) + +[id="{version}-plugins-{type}s-{plugin}-field_namespace"] +===== `field_namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_namespace"` + +The name of the field used to set a different namespace per event +Note: Only one namespace can be sent to CloudWatch per API call +so setting different namespaces will increase the number of API calls +and those cost money. + +[id="{version}-plugins-{type}s-{plugin}-field_unit"] +===== `field_unit` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_unit"` + +The name of the field used to set the unit on an event metric + +[id="{version}-plugins-{type}s-{plugin}-field_value"] +===== `field_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_value"` + +The name of the field used to set the value (float) on an event metric + +[id="{version}-plugins-{type}s-{plugin}-metricname"] +===== `metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The default metric name to use for events which do not have a `CW_metricname` field. +Beware: If this is provided then all events which pass through this output will be aggregated and +sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you +will probably want to also restrict events from passing through this output using event +type, tag, and field matching + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Logstash"` + +The default namespace to use for events which do not have a `CW_namespace` field + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue_size"] +===== `queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule +Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-timeframe"] +===== `timeframe` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +Constants +aggregate_key members +Units +How often to send data to CloudWatch +This does not affect the event timestamps, events will always have their +actual timestamp (to-the-minute) sent to CloudWatch. + +We only call the API if there is data to send. + +See the Rufus Scheduler docs for an https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler[explanation of allowed values] + +[id="{version}-plugins-{type}s-{plugin}-unit"] +===== `unit` + + * Value can be any of: `Seconds`, `Microseconds`, `Milliseconds`, `Bytes`, `Kilobytes`, `Megabytes`, `Gigabytes`, `Terabytes`, `Bits`, `Kilobits`, `Megabits`, `Gigabits`, `Terabits`, `Percent`, `Count`, `Bytes/Second`, `Kilobytes/Second`, `Megabytes/Second`, `Gigabytes/Second`, `Terabytes/Second`, `Bits/Second`, `Kilobits/Second`, `Megabits/Second`, `Gigabits/Second`, `Terabits/Second`, `Count/Second`, `None` + * Default value is `"Count"` + +The default unit to use for events which do not have a `CW_unit` field +If you set this option you should probably set the "value" option along with it + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-value"] +===== `value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1"` + +The default value to use for events which do not have a `CW_value` field +If provided, this must be a string which can be converted to a float, for example... + "1", "2.34", ".5", and "0.67" +If you set this option you should probably set the `unit` option along with it + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/cloudwatch-v7.2.1.asciidoc b/docs/versioned-plugins/outputs/cloudwatch-v7.2.1.asciidoc new file mode 100644 index 000000000..6b5c78218 --- /dev/null +++ b/docs/versioned-plugins/outputs/cloudwatch-v7.2.1.asciidoc @@ -0,0 +1,331 @@ +:integration: aws +:plugin: cloudwatch +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Cloudwatch output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This output lets you aggregate and send metric data to AWS CloudWatch + +==== Summary: +This plugin is intended to be used on a logstash indexer agent (but that +is not the only way, see below.) In the intended scenario, one cloudwatch +output plugin is configured, on the logstash indexer node, with just AWS API +credentials, and possibly a region and/or a namespace. The output looks +for fields present in events, and when it finds them, it uses them to +calculate aggregate statistics. If the `metricname` option is set in this +output, then any events which pass through it will be aggregated & sent to +CloudWatch, but that is not recommended. The intended use is to NOT set the +metricname option here, and instead to add a `CW_metricname` field (and other +fields) to only the events you want sent to CloudWatch. + +When events pass through this output they are queued for background +aggregation and sending, which happens every minute by default. The +queue has a maximum size, and when it is full aggregated statistics will be +sent to CloudWatch ahead of schedule. Whenever this happens a warning +message is written to logstash's log. If you see this you should increase +the `queue_size` configuration option to avoid the extra API calls. The queue +is emptied every time we send data to CloudWatch. + +Note: when logstash is stopped the queue is destroyed before it can be processed. +This is a known limitation of logstash and will hopefully be addressed in a +future version. + +==== Details: +There are two ways to configure this plugin, and they can be used in +combination: event fields & per-output defaults + +Event Field configuration... +You add fields to your events in inputs & filters and this output reads +those fields to aggregate events. The names of the fields read are +configurable via the `field_*` options. + +Per-output defaults... +You set universal defaults in this output plugin's configuration, and +if an event does not have a field for that option then the default is +used. + +Notice, the event fields take precedence over the per-output defaults. + +At a minimum events must have a "metric name" to be sent to CloudWatch. +This can be achieved either by providing a default here OR by adding a +`CW_metricname` field. By default, if no other configuration is provided +besides a metric name, then events will be counted (Unit: Count, Value: 1) +by their metric name (either a default or from their `CW_metricname` field) + +Other fields which can be added to events to modify the behavior of this +plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and +`CW_dimensions`. All of these field names are configurable in +this output. You can also set per-output defaults for any of them. +See below for details. + +Read more about http://aws.amazon.com/cloudwatch/[AWS CloudWatch], +and the specific of API endpoint this output uses, +http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html[PutMetricData] + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Cloudwatch Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dimensions>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-field_dimensions>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_unit>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-field_value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metricname>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-timeframe>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-unit>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", "Gigabytes", "Terabytes", "Bits", "Kilobits", "Megabits", "Gigabits", "Terabits", "Percent", "Count", "Bytes/Second", "Kilobytes/Second", "Megabytes/Second", "Gigabytes/Second", "Terabytes/Second", "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-value>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `20` + +How many data points can be given in one call to the CloudWatch API + +[id="{version}-plugins-{type}s-{plugin}-dimensions"] +===== `dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field + +[id="{version}-plugins-{type}s-{plugin}-field_dimensions"] +===== `field_dimensions` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_dimensions"` + +The name of the field used to set the dimensions on an event metric +The field named here, if present in an event, must have an array of +one or more key & value pairs, for example... + `add_field => [ "CW_dimensions", "Environment", "CW_dimensions", "prod" ]` +or, equivalently... + `add_field => [ "CW_dimensions", "Environment" ]` + `add_field => [ "CW_dimensions", "prod" ]` + +[id="{version}-plugins-{type}s-{plugin}-field_metricname"] +===== `field_metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_metricname"` + +The name of the field used to set the metric name on an event +The author of this plugin recommends adding this field to events in inputs & +filters rather than using the per-output default setting so that one output +plugin on your logstash indexer can serve all events (which of course had +fields set on your logstash shippers.) + +[id="{version}-plugins-{type}s-{plugin}-field_namespace"] +===== `field_namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_namespace"` + +The name of the field used to set a different namespace per event +Note: Only one namespace can be sent to CloudWatch per API call +so setting different namespaces will increase the number of API calls +and those cost money. + +[id="{version}-plugins-{type}s-{plugin}-field_unit"] +===== `field_unit` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_unit"` + +The name of the field used to set the unit on an event metric + +[id="{version}-plugins-{type}s-{plugin}-field_value"] +===== `field_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"CW_value"` + +The name of the field used to set the value (float) on an event metric + +[id="{version}-plugins-{type}s-{plugin}-metricname"] +===== `metricname` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The default metric name to use for events which do not have a `CW_metricname` field. +Beware: If this is provided then all events which pass through this output will be aggregated and +sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you +will probably want to also restrict events from passing through this output using event +type, tag, and field matching + +[id="{version}-plugins-{type}s-{plugin}-namespace"] +===== `namespace` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"Logstash"` + +The default namespace to use for events which do not have a `CW_namespace` field + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue_size"] +===== `queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule +Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-timeframe"] +===== `timeframe` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1m"` + +Constants +aggregate_key members +Units +How often to send data to CloudWatch +This does not affect the event timestamps, events will always have their +actual timestamp (to-the-minute) sent to CloudWatch. + +We only call the API if there is data to send. + +See the Rufus Scheduler docs for an https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler[explanation of allowed values] + +[id="{version}-plugins-{type}s-{plugin}-unit"] +===== `unit` + + * Value can be any of: `Seconds`, `Microseconds`, `Milliseconds`, `Bytes`, `Kilobytes`, `Megabytes`, `Gigabytes`, `Terabytes`, `Bits`, `Kilobits`, `Megabits`, `Gigabits`, `Terabits`, `Percent`, `Count`, `Bytes/Second`, `Kilobytes/Second`, `Megabytes/Second`, `Gigabytes/Second`, `Terabytes/Second`, `Bits/Second`, `Kilobits/Second`, `Megabits/Second`, `Gigabits/Second`, `Terabits/Second`, `Count/Second`, `None` + * Default value is `"Count"` + +The default unit to use for events which do not have a `CW_unit` field +If you set this option you should probably set the "value" option along with it + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-value"] +===== `value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"1"` + +The default value to use for events which do not have a `CW_value` field +If provided, this must be a string which can be converted to a float, for example... + "1", "2.34", ".5", and "0.67" +If you set this option you should probably set the `unit` option along with it + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/csv-index.asciidoc b/docs/versioned-plugins/outputs/csv-index.asciidoc index 98c32e0db..fd0958efb 100644 --- a/docs/versioned-plugins/outputs/csv-index.asciidoc +++ b/docs/versioned-plugins/outputs/csv-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2023-12-19 | <> | 2023-06-14 | <> | 2020-01-14 | <> | 2018-04-06 @@ -13,6 +14,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::csv-v3.0.10.asciidoc[] include::csv-v3.0.9.asciidoc[] include::csv-v3.0.8.asciidoc[] include::csv-v3.0.7.asciidoc[] diff --git a/docs/versioned-plugins/outputs/csv-v3.0.10.asciidoc b/docs/versioned-plugins/outputs/csv-v3.0.10.asciidoc new file mode 100644 index 000000000..12cbe64b0 --- /dev/null +++ b/docs/versioned-plugins/outputs/csv-v3.0.10.asciidoc @@ -0,0 +1,183 @@ +:plugin: csv +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.0.10 +:release_date: 2023-12-19 +:changelog_url: https://github.com/logstash-plugins/logstash-output-csv/blob/v3.0.10/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Csv output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +CSV output. + +Write events to disk in CSV or other delimited format +Based on the file output, many config values are shared +Uses the Ruby csv library internally + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Csv Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-create_if_deleted>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-csv_options>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-dir_mode>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|Yes +| <<{version}-plugins-{type}s-{plugin}-file_mode>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-filename_failure>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-flush_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-gzip>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-spreadsheet_safe>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-create_if_deleted"] +===== `create_if_deleted` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +If the configured file is deleted, but an event is handled by the plugin, +the plugin will recreate the file. Default => true + +[id="{version}-plugins-{type}s-{plugin}-csv_options"] +===== `csv_options` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Options for CSV output. This is passed directly to the Ruby stdlib to_csv function. +Full documentation is available on the http://ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/index.html[Ruby CSV documentation page]. +A typical use case would be to use alternative column or row separators eg: `csv_options => {"col_sep" => "\t" "row_sep" => "\r\n"}` gives tab separated data with windows line endings + +[id="{version}-plugins-{type}s-{plugin}-dir_mode"] +===== `dir_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `-1` + +Dir access mode to use. Note that due to the bug in jruby system umask +is ignored on linux: https://github.com/jruby/jruby/issues/3426 +Setting it to -1 uses default OS value. +Example: `"dir_mode" => 0750` + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +The field names from the event that should be written to the CSV file. +Fields are written to the CSV in the same order as the array. +If a field does not exist on the event, an empty string will be written. +Supports field reference syntax eg: `fields => ["field1", "[nested][field]"]`. + +[id="{version}-plugins-{type}s-{plugin}-file_mode"] +===== `file_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `-1` + +File access mode to use. Note that due to the bug in jruby system umask +is ignored on linux: https://github.com/jruby/jruby/issues/3426 +Setting it to -1 uses default OS value. +Example: `"file_mode" => 0640` + +[id="{version}-plugins-{type}s-{plugin}-filename_failure"] +===== `filename_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"_filepath_failures"` + +If the generated path is invalid, the events will be saved +into this file and inside the defined path. + +[id="{version}-plugins-{type}s-{plugin}-flush_interval"] +===== `flush_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Flush interval (in seconds) for flushing writes to log files. +0 will flush on every message. + +[id="{version}-plugins-{type}s-{plugin}-gzip"] +===== `gzip` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Gzip the output stream before writing to disk. + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This output writes events to files on disk. You can use fields +from the event as parts of the filename and/or path. + +By default, this output writes one event per line in **json** format. +You can customise the line format using the `line` codec like: + +[source,ruby] +----- +output { + file { + path => ... + codec => line { format => "custom format: %{message}"} + } +} +----- + +The path to the file to write. Event fields can be used here, +like `/var/log/logstash/%{host}/%{application}` +One may also utilize the path option for date-based log +rotation via the joda time format. This will use the event +timestamp. +E.g.: `path => "./test-%{+YYYY-MM-dd}.txt"` to create +`./test-2013-05-29.txt` + +If you use an absolute path you cannot start with a dynamic string. +E.g: `/%{myfield}/`, `/test-%{myfield}/` are not valid paths + +[id="{version}-plugins-{type}s-{plugin}-spreadsheet_safe"] +===== `spreadsheet_safe` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Option to not escape/munge string values. Please note turning off this option +may not make the values safe in your spreadsheet application + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/datadog_metrics-index.asciidoc b/docs/versioned-plugins/outputs/datadog_metrics-index.asciidoc index fb3d58f3e..2f054bb97 100644 --- a/docs/versioned-plugins/outputs/datadog_metrics-index.asciidoc +++ b/docs/versioned-plugins/outputs/datadog_metrics-index.asciidoc @@ -5,12 +5,16 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-10-25 +| <> | 2024-08-09 | <> | 2023-05-31 | <> | 2018-04-06 | <> | 2017-08-21 | <> | 2017-06-23 |======================================================================= +include::datadog_metrics-v3.0.7.asciidoc[] +include::datadog_metrics-v3.0.6.asciidoc[] include::datadog_metrics-v3.0.5.asciidoc[] include::datadog_metrics-v3.0.4.asciidoc[] include::datadog_metrics-v3.0.2.asciidoc[] diff --git a/docs/versioned-plugins/outputs/datadog_metrics-v3.0.6.asciidoc b/docs/versioned-plugins/outputs/datadog_metrics-v3.0.6.asciidoc new file mode 100644 index 000000000..8dfaaa90d --- /dev/null +++ b/docs/versioned-plugins/outputs/datadog_metrics-v3.0.6.asciidoc @@ -0,0 +1,142 @@ +:plugin: datadog_metrics +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.0.6 +:release_date: 2024-08-09 +:changelog_url: https://github.com/logstash-plugins/logstash-output-datadog_metrics/blob/v3.0.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Datadog_metrics output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output lets you send metrics to +DataDogHQ based on Logstash events. +Default `queue_size` and `timeframe` are low in order to provide near realtime alerting. +If you do not use Datadog for alerting, consider raising these thresholds. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Datadog_metrics Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|Yes +| <<{version}-plugins-{type}s-{plugin}-api_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dd_tags>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-device>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metric_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metric_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["gauge", "counter", "%{metric_type}"]`|No +| <<{version}-plugins-{type}s-{plugin}-metric_value>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-timeframe>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Your DatadogHQ API key. https://app.datadoghq.com/account/settings#api + +[id="{version}-plugins-{type}s-{plugin}-api_url"] +===== `api_url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https://api.datadoghq.com/api/v1/series"` + +Set the api endpoint for Datadog EU Site users + +[id="{version}-plugins-{type}s-{plugin}-dd_tags"] +===== `dd_tags` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +Set any custom tags for this event, +default are the Logstash tags if any. + +[id="{version}-plugins-{type}s-{plugin}-device"] +===== `device` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{metric_device}"` + +The name of the device that produced the metric. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{host}"` + +The name of the host that produced the metric. + +[id="{version}-plugins-{type}s-{plugin}-metric_name"] +===== `metric_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{metric_name}"` + +The name of the time series. + +[id="{version}-plugins-{type}s-{plugin}-metric_type"] +===== `metric_type` + + * Value can be any of: `gauge`, `counter`, `%{metric_type}` + * Default value is `"%{metric_type}"` + +The type of the metric. + +[id="{version}-plugins-{type}s-{plugin}-metric_value"] +===== `metric_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{metric_value}"` + +The value. + +[id="{version}-plugins-{type}s-{plugin}-queue_size"] +===== `queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +How many events to queue before flushing to Datadog +prior to schedule set in `@timeframe` + +[id="{version}-plugins-{type}s-{plugin}-timeframe"] +===== `timeframe` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +How often (in seconds) to flush queued events to Datadog + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/datadog_metrics-v3.0.7.asciidoc b/docs/versioned-plugins/outputs/datadog_metrics-v3.0.7.asciidoc new file mode 100644 index 000000000..b82c612c0 --- /dev/null +++ b/docs/versioned-plugins/outputs/datadog_metrics-v3.0.7.asciidoc @@ -0,0 +1,142 @@ +:plugin: datadog_metrics +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.0.7 +:release_date: 2024-10-25 +:changelog_url: https://github.com/logstash-plugins/logstash-output-datadog_metrics/blob/v3.0.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Datadog_metrics output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output lets you send metrics to +DataDogHQ based on Logstash events. +Default `queue_size` and `timeframe` are low in order to provide near realtime alerting. +If you do not use Datadog for alerting, consider raising these thresholds. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Datadog_metrics Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|Yes +| <<{version}-plugins-{type}s-{plugin}-api_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dd_tags>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-device>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metric_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metric_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["gauge", "counter", "%{metric_type}"]`|No +| <<{version}-plugins-{type}s-{plugin}-metric_value>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-timeframe>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Your DatadogHQ API key. https://app.datadoghq.com/account/settings#api + +[id="{version}-plugins-{type}s-{plugin}-api_url"] +===== `api_url` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https://api.datadoghq.com/api/v1/series"` + +Set the api endpoint for Datadog EU Site users + +[id="{version}-plugins-{type}s-{plugin}-dd_tags"] +===== `dd_tags` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * There is no default value for this setting. + +Set any custom tags for this event, +default are the Logstash tags if any. + +[id="{version}-plugins-{type}s-{plugin}-device"] +===== `device` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{metric_device}"` + +The name of the device that produced the metric. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{host}"` + +The name of the host that produced the metric. + +[id="{version}-plugins-{type}s-{plugin}-metric_name"] +===== `metric_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{metric_name}"` + +The name of the time series. + +[id="{version}-plugins-{type}s-{plugin}-metric_type"] +===== `metric_type` + + * Value can be any of: `gauge`, `counter`, `%{metric_type}` + * Default value is `"%{metric_type}"` + +The type of the metric. + +[id="{version}-plugins-{type}s-{plugin}-metric_value"] +===== `metric_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%{metric_value}"` + +The value. + +[id="{version}-plugins-{type}s-{plugin}-queue_size"] +===== `queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +How many events to queue before flushing to Datadog +prior to schedule set in `@timeframe` + +[id="{version}-plugins-{type}s-{plugin}-timeframe"] +===== `timeframe` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +How often (in seconds) to flush queued events to Datadog + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/elastic_app_search-index.asciidoc b/docs/versioned-plugins/outputs/elastic_app_search-index.asciidoc index a89dcf829..1671c6ac4 100644 --- a/docs/versioned-plugins/outputs/elastic_app_search-index.asciidoc +++ b/docs/versioned-plugins/outputs/elastic_app_search-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-01-03 | <> | 2023-11-07 | <> | 2022-01-28 | <> | 2022-01-24 @@ -18,6 +19,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2019-06-12 |======================================================================= +include::elastic_app_search-v3.0.1.asciidoc[] include::elastic_app_search-v3.0.0.asciidoc[] include::elastic_app_search-v2.2.1.asciidoc[] include::elastic_app_search-v2.2.0.asciidoc[] diff --git a/docs/versioned-plugins/outputs/elastic_app_search-v3.0.1.asciidoc b/docs/versioned-plugins/outputs/elastic_app_search-v3.0.1.asciidoc new file mode 100644 index 000000000..5049bcf98 --- /dev/null +++ b/docs/versioned-plugins/outputs/elastic_app_search-v3.0.1.asciidoc @@ -0,0 +1,236 @@ +:integration: elastic_enterprise_search +:plugin: elastic_app_search +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.0.1 +:release_date: 2025-01-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/blob/v3.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elastic App Search output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This output lets you send events to the https://www.elastic.co/app-search[Elastic App Search] solution, +both the https://www.elastic.co/downloads/app-search[self-managed] or +the https://www.elastic.co/cloud/app-search-service[managed] service. +On receiving a batch of events from the Logstash pipeline, the plugin +converts the events into documents and uses the App Search bulk API +to index multiple events in one request. + +App Search doesn't allow fields to begin with `@timestamp`. +By default the `@timestamp` and `@version` fields will be removed from +each event before the event is sent to App Search. If you want to keep +the `@timestamp` field, you can use the +<<{version}-plugins-{type}s-{plugin}-timestamp_destination,timestamp_destination>> option +to store the timestamp in a different field. + +NOTE: This gem does not support codec customization. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== AppSearch Output configuration options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|Yes +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-engine>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-timestamp_destination>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value + +The private API Key with write permissions. Visit the App Search API keys reference [page](https://www.elastic.co/guide/en/app-search/current/authentication.html#authentication-api-keys) for more information. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value + +The id for app search documents. This can be an interpolated value +like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten. + +[id="{version}-plugins-{type}s-{plugin}-engine"] +===== `engine` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value + +The name of the search engine you created in App Search, an information +repository that includes the indexed document records. +The `engine` field supports +{logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to +allow the engine name to be derived from a field value from each event, for +example `engine-%{engine_name}`. + +Invalid engine names cause ingestion to stop until the field value can be resolved into a valid engine name. +This situation can happen if the interpolated field value resolves to a value without a matching engine, +or, if the field is missing from the event and cannot be resolved at all. + +TIP: Consider adding a "default" engine type in the configuration to catch errors if +the field is missing from the event. + +Example: + + + +[source,ruby] +---------------------------------- +input { + stdin { + codec => json + } +} + +filter { + if ![engine_name] { + mutate { + add_field => {"engine_name" => "default"} + } + } +} + +output { + elastic_app_search { + engine => "engine_%{[engine_name]}" + } +} +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` +* Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` +* Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. +`'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value can be any of: `full`, `none` +* Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-timestamp_destination"] +===== `timestamp_destination` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value + +Where to move the value from the `@timestamp` field. + +All Logstash events contain a `@timestamp` field. +App Search doesn't support fields starting with `@timestamp`, and +by default, the `@timestamp` field will be deleted. + +To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied. + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `http://localhost:3002` + +The value of the API endpoint in the form of a URL. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elastic_workplace_search-index.asciidoc b/docs/versioned-plugins/outputs/elastic_workplace_search-index.asciidoc index 489e78373..d97a60a56 100644 --- a/docs/versioned-plugins/outputs/elastic_workplace_search-index.asciidoc +++ b/docs/versioned-plugins/outputs/elastic_workplace_search-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-01-03 | <> | 2023-11-07 | <> | 2022-01-28 | <> | 2022-01-24 @@ -13,6 +14,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2021-05-18 |======================================================================= +include::elastic_workplace_search-v3.0.1.asciidoc[] include::elastic_workplace_search-v3.0.0.asciidoc[] include::elastic_workplace_search-v2.2.1.asciidoc[] include::elastic_workplace_search-v2.2.0.asciidoc[] diff --git a/docs/versioned-plugins/outputs/elastic_workplace_search-v3.0.1.asciidoc b/docs/versioned-plugins/outputs/elastic_workplace_search-v3.0.1.asciidoc new file mode 100644 index 000000000..c6fa3c1a6 --- /dev/null +++ b/docs/versioned-plugins/outputs/elastic_workplace_search-v3.0.1.asciidoc @@ -0,0 +1,245 @@ +:integration: elastic_enterprise_search +:plugin: elastic_workplace_search +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.0.1 +:release_date: 2025-01-03 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/blob/v3.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elastic Workplace Search output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This output lets you send events to the https://www.elastic.co/workplace-search[Elastic Workplace Search] solution. +On receiving a batch of events from the Logstash pipeline, the plugin +converts the events into documents and uses the Workplace Search bulk API +to index multiple events in one request. + +Workplace Search doesn't allow fields to begin with `@timestamp`. +By default the `@timestamp` and `@version` fields will be removed from +each event before the event is sent to Workplace Search. If you want to keep +the `@timestamp` field, you can use the +<<{version}-plugins-{type}s-{plugin}-timestamp_destination,timestamp_destination>> option +to store the timestamp in a different field. + +NOTE: This gem does not support codec customization. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Workplace Search Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_token>> |{logstash-ref}/configuration-file-structure.html#password[password]|Yes +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-source>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-timestamp_destination>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_token"] +===== `access_token` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value + +The source access token. Visit the source overview page in the Workplace Search dashboard to find the token associated with your source. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value + +The id for workplace search documents. This can be an interpolated value +like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten. + +[id="{version}-plugins-{type}s-{plugin}-source"] +===== `source` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value + +The ID of the source you created in Workplace Search. +The `source` field supports +{logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to +allow the source ID to be derived from a field value from each event, for +example `%{source_id}`. + +Invalid source IDs cause ingestion to stop until the field value can be resolved into a valid source ID. +This situation can happen if the interpolated field value resolves to a value without a matching source, +or, if the field is missing from the event and cannot be resolved at all. + +TIP: Consider adding a "default" source type in the configuration to catch errors if +the field is missing from the event. + +Example: + + + +[source,ruby] +---------------------------------- +input { + stdin { + codec => json + } +} + +filter { + if ![source_id] { + mutate { + add_field => {"source_id" => "default"} + } + } +} + +output { + elastic_workplace_search { + source => "%{[source_id]}" + access_token => "abracadabra" + url => "http://workplace.search.com:3002" + } +} +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` +* Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` +* Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. +`'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value can be any of: `full`, `none` +* Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-timestamp_destination"] +===== `timestamp_destination` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value + +Where to move the value from the `@timestamp` field. + +All Logstash events contain a `@timestamp` field. +Workplace Search doesn't support fields starting with `@timestamp`, and +by default, the `@timestamp` field will be deleted. + +To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied. + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `http://localhost:3002` + +The value of the API endpoint in the form of a URL. + +**Examples** + +On premise instance: + +`http://workplace.company.com:3002` + +Elastic Cloud instance: + +`https://7c455f508468426cb53912be65548117.ent-search.eu-west-1.aws.cloud.es.io` + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-index.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-index.asciidoc index 7e7c04140..1dae87ac1 100644 --- a/docs/versioned-plugins/outputs/elasticsearch-index.asciidoc +++ b/docs/versioned-plugins/outputs/elasticsearch-index.asciidoc @@ -5,6 +5,25 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-07-29 +| <> | 2025-07-09 +| <> | 2025-07-07 +| <> | 2025-04-17 +| <> | 2025-01-23 +| <> | 2025-01-14 +| <> | 2024-12-24 +| <> | 2025-08-01 +| <> | 2025-01-28 +| <> | 2025-01-15 +| <> | 2024-12-09 +| <> | 2024-09-13 +| <> | 2024-09-13 +| <> | 2024-06-25 +| <> | 2024-04-30 +| <> | 2024-04-02 +| <> | 2024-03-22 +| <> | 2024-03-19 +| <> | 2023-12-13 | <> | 2023-11-21 | <> | 2023-11-16 | <> | 2023-11-10 @@ -42,6 +61,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2022-08-30 | <> | 2022-05-24 | <> | 2022-04-04 +| <> | 2024-03-20 | <> | 2022-01-19 | <> | 2022-01-13 | <> | 2021-12-27 @@ -119,6 +139,25 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-26 |======================================================================= +include::elasticsearch-v12.0.6.asciidoc[] +include::elasticsearch-v12.0.5.asciidoc[] +include::elasticsearch-v12.0.4.asciidoc[] +include::elasticsearch-v12.0.3.asciidoc[] +include::elasticsearch-v12.0.2.asciidoc[] +include::elasticsearch-v12.0.1.asciidoc[] +include::elasticsearch-v12.0.0.asciidoc[] +include::elasticsearch-v11.22.13.asciidoc[] +include::elasticsearch-v11.22.12.asciidoc[] +include::elasticsearch-v11.22.11.asciidoc[] +include::elasticsearch-v11.22.10.asciidoc[] +include::elasticsearch-v11.22.9.asciidoc[] +include::elasticsearch-v11.22.8.asciidoc[] +include::elasticsearch-v11.22.7.asciidoc[] +include::elasticsearch-v11.22.6.asciidoc[] +include::elasticsearch-v11.22.5.asciidoc[] +include::elasticsearch-v11.22.4.asciidoc[] +include::elasticsearch-v11.22.3.asciidoc[] +include::elasticsearch-v11.22.2.asciidoc[] include::elasticsearch-v11.22.1.asciidoc[] include::elasticsearch-v11.22.0.asciidoc[] include::elasticsearch-v11.21.0.asciidoc[] @@ -156,6 +195,7 @@ include::elasticsearch-v11.8.0.asciidoc[] include::elasticsearch-v11.7.0.asciidoc[] include::elasticsearch-v11.6.0.asciidoc[] include::elasticsearch-v11.5.0.asciidoc[] +include::elasticsearch-v11.4.2.asciidoc[] include::elasticsearch-v11.4.1.asciidoc[] include::elasticsearch-v11.4.0.asciidoc[] include::elasticsearch-v11.3.3.asciidoc[] diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.10.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.10.asciidoc new file mode 100644 index 000000000..8fd6711c1 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.10.asciidoc @@ -0,0 +1,1422 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.10 +:release_date: 2024-12-09 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.10/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.11.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.11.asciidoc new file mode 100644 index 000000000..0f041d48c --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.11.asciidoc @@ -0,0 +1,1422 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.11 +:release_date: 2025-01-15 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.11/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.12.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.12.asciidoc new file mode 100644 index 000000000..24d17b990 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.12.asciidoc @@ -0,0 +1,1437 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.12 +:release_date: 2025-01-28 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.12/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.13.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.13.asciidoc new file mode 100644 index 000000000..776dae663 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.13.asciidoc @@ -0,0 +1,1437 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.13 +:release_date: 2025-08-01 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.13/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.2.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.2.asciidoc new file mode 100644 index 000000000..43ff192bf --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.2.asciidoc @@ -0,0 +1,1406 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.2 +:release_date: 2023-12-13 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-logstash`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The index to write events to. This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.3.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.3.asciidoc new file mode 100644 index 000000000..95c23e24a --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.3.asciidoc @@ -0,0 +1,1411 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.3 +:release_date: 2024-03-19 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-logstash`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The index to write events to. This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.4.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.4.asciidoc new file mode 100644 index 000000000..0789b21c0 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.4.asciidoc @@ -0,0 +1,1412 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.4 +:release_date: 2024-03-22 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-logstash`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The index to write events to. This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.5.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.5.asciidoc new file mode 100644 index 000000000..37b452264 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.5.asciidoc @@ -0,0 +1,1412 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.5 +:release_date: 2024-04-02 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The index to write events to. This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.6.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.6.asciidoc new file mode 100644 index 000000000..6f2bd12ac --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.6.asciidoc @@ -0,0 +1,1414 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.6 +:release_date: 2024-04-30 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.7.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.7.asciidoc new file mode 100644 index 000000000..7f551d915 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.7.asciidoc @@ -0,0 +1,1422 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.7 +:release_date: 2024-06-25 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.8.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.8.asciidoc new file mode 100644 index 000000000..ac2219b61 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.8.asciidoc @@ -0,0 +1,1422 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.8 +:release_date: 2024-09-13 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.22.9.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.22.9.asciidoc new file mode 100644 index 000000000..e85970416 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.22.9.asciidoc @@ -0,0 +1,1422 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.22.9 +:release_date: 2024-09-13 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.22.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> and the <<{version}-plugins-{type}s-{plugin}-deprecated-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Output Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[11.14.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v11.4.2.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v11.4.2.asciidoc new file mode 100644 index 000000000..80bc6757a --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v11.4.2.asciidoc @@ -0,0 +1,1142 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.2 +:release_date: 2024-03-20 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v11.4.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +================================================================================ +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +================================================================================ + +===== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +The data stream options are recommended for indexing time series datasets (such +as logs, metrics, and events) into {es}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-logstash`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== Index Lifecycle Management + +[NOTE] +The Index Lifecycle Management feature requires plugin version `9.3.1` or higher. + +[NOTE] +This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license + +Logstash can use {ref}/index-lifecycle-management.html[Index Lifecycle +Management] to automate the management of indices over time. + +The use of Index Lifecycle Management is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports ILM, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable ILM. + +This will overwrite the index settings and adjust the Logstash template to write +the necessary settings for the template to support index lifecycle management, +including the index policy and rollover alias to be used. + +Logstash will create a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +NOTE: Custom ILM policies must already exist on the Elasticsearch cluster before they can be used. + +NOTE: If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template + +NOTE: If the index property is supplied in the output definition, it will be overwritten by the rollover alias. + + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin supports request compression, and handles compressed responses +from Elasticsearch. + +To enable request compression, use the <<{version}-plugins-{type}s-{plugin}-http_compression>> +setting on this plugin. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports the following configuration options plus the +<<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_verification>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the `ssl` option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to perform the _bulk requests to +this defaults to a concatenation of the path parameter and "_bulk" + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The .cer or .pem file to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`: provides defaults that are compatible with v1 of the Elastic Common Schema +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Set the Elasticsearch errors in the whitelist that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `document_already_exists_exception`. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `http_compression`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable gzip compression on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The index to write events to. This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] for the index pattern from event timestamp. + + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either .jks or .p12 + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => +"%{[@metadata][pipeline]}"`. The pipeline parameter won't be set if the value +resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this +unspecified will use whatever scheme is specified in the URLs listed in 'hosts'. +If no explicit protocol is specified plain HTTP will be used. If SSL is +explicitly disabled here the plugin will refuse to start if an HTTPS URL is +given in 'hosts' + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_verification"] +===== `ssl_certificate_verification` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Option to validate the server's certificate. Disabling this severely compromises security. +For more information on disabling certificate verification please read +https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either .jks or .p12. +Use either `:truststore` or `:cacert`. + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v12.0.0.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v12.0.0.asciidoc new file mode 100644 index 000000000..cfee608bb --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v12.0.0.asciidoc @@ -0,0 +1,1350 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v12.0.0 +:release_date: 2024-12-24 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v12.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| truststore | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v12.0.1.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v12.0.1.asciidoc new file mode 100644 index 000000000..1ce2167d3 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v12.0.1.asciidoc @@ -0,0 +1,1350 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v12.0.1 +:release_date: 2025-01-14 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v12.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. It will be removed in the next major version of +Logstash. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used; +- for elasticsearch clusters 6.x: the value of 'doc' will be used; +- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. +For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will +be added to the hosts list, excluding master-only nodes. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| truststore | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v12.0.2.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v12.0.2.asciidoc new file mode 100644 index 000000000..5f70aabfc --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v12.0.2.asciidoc @@ -0,0 +1,1360 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v12.0.2 +:release_date: 2025-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v12.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| truststore | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v12.0.3.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v12.0.3.asciidoc new file mode 100644 index 000000000..0aa683eb0 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v12.0.3.asciidoc @@ -0,0 +1,1360 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v12.0.3 +:release_date: 2025-04-17 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v12.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== Hosted {es} Service on Elastic Cloud + +{ess-leadin} + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| truststore | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v12.0.4.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v12.0.4.asciidoc new file mode 100644 index 000000000..9ceacc750 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v12.0.4.asciidoc @@ -0,0 +1,1361 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v12.0.4 +:release_date: 2025-07-07 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v12.0.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== {ls} to {ech} + +You can run Elasticsearch on your own hardware or use Elastic Cloud Hosted, available on AWS, GCP, and Azure. +Try Elastic Cloud Hosted for free: https://cloud.elastic.co/registration. + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry Policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ Policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. See +{logstash-ref}/dead-letter-queues.html[dead-letter-queues] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| truststore | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v12.0.5.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v12.0.5.asciidoc new file mode 100644 index 000000000..943c2c663 --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v12.0.5.asciidoc @@ -0,0 +1,1361 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v12.0.5 +:release_date: 2025-07-09 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v12.0.5/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== {ls} to {ech} + +You can run Elasticsearch on your own hardware or use Elastic Cloud Hosted, available on AWS, GCP, and Azure. +Try Elastic Cloud Hosted for free: https://cloud.elastic.co/registration. + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. +Check out https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html[Dead letter queues (DLQ)] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| truststore | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/elasticsearch-v12.0.6.asciidoc b/docs/versioned-plugins/outputs/elasticsearch-v12.0.6.asciidoc new file mode 100644 index 000000000..f84e9b94b --- /dev/null +++ b/docs/versioned-plugins/outputs/elasticsearch-v12.0.6.asciidoc @@ -0,0 +1,1361 @@ +:plugin: elasticsearch +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v12.0.6 +:release_date: 2025-07-29 +:changelog_url: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/v12.0.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Elasticsearch provides near real-time search and analytics for all types of +data. The Elasticsearch output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on +the website landing page or in the {ref}[Elasticsearch documentation]. + +.Compatibility Note +[NOTE] +===== +When connected to Elasticsearch 7.x, modern versions of this plugin +don't use the document-type when inserting documents, unless the user +explicitly sets <<{version}-plugins-{type}s-{plugin}-document_type>>. + +If you are using an earlier version of Logstash and wish to connect to +Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it +picks up changes to the Elasticsearch index template. + +If you are using a custom <<{version}-plugins-{type}s-{plugin}-template>>, +ensure your template uses the `_doc` document-type before +connecting to Elasticsearch 7.x. +===== + +[id="{version}-plugins-{type}s-{plugin}-serverless"] +==== {ls} to {es-serverless} + +You can use this plugin to send your {ls} data to {es-serverless}. +Some differences to note between {es-serverless} and self-managed {es}: + +* Use *API keys* to access {serverless-full} from {ls}. +Any user-based security settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). +Any {ilm-init} settings in your <<{version}-plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors. +* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}. + +.Known issue for {ls} to {es-serverless} +**** +The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. +Set the value to port :443 instead. +**** + +For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. + +[id="{version}-plugins-{type}s-{plugin}-ess"] +==== {ls} to {ech} + +You can run Elasticsearch on your own hardware or use Elastic Cloud Hosted, available on AWS, GCP, and Azure. +Try Elastic Cloud Hosted for free: https://cloud.elastic.co/registration. + +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>>. +By having an ECS-compatible template in place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. + +[id="{version}-plugins-{type}s-{plugin}-data-streams"] +==== Data streams + +The {es} output plugin can store both time series datasets (such +as logs, events, and metrics) and non-time series data in Elasticsearch. + +Use the data stream options for indexing time series datasets (such +as logs, metrics, and events) into {es} and {es-serverless}: + +* <<{version}-plugins-{type}s-{plugin}-data_stream>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> +* <<{version}-plugins-{type}s-{plugin}-data_stream_type>> + +IMPORTANT: <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-ds-examples"] +===== Data stream configuration examples + +**Example: Basic default configuration** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + } +} +----- + +This example shows the minimal settings for processing data streams. Events +with `data_stream.*`` fields are routed to the appropriate data streams. If the +fields are missing, routing defaults to `logs-generic-default`. + +**Example: Customize data stream name** + +[source,sh] +----- +output { + elasticsearch { + hosts => "hostname" + data_stream => "true" + data_stream_type => "metrics" + data_stream_dataset => "foo" + data_stream_namespace => "bar" + } +} +----- + + +==== Writing to different indices: best practices + +NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` +and when using `ilm_rollover_alias`. + +If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can: + + * use different Elasticsearch outputs, each one with a different value for the `index` parameter + * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter + +Each Elasticsearch output is a new client connected to the cluster: + + * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients) + * it has an associated connection pool + +In order to minimize the number of open connections to Elasticsearch, maximize +the bulk size and reduce the number of "small" bulk requests (which could easily +fill up the queue), it is usually more efficient to have a single Elasticsearch +output. + +Example: +[source,ruby] + output { + elasticsearch { + index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}" + } + } + +**What to do in case there is no field in the event containing the destination index prefix?** + +You can use the `mutate` filter and conditionals to add a +{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field] +to set the destination index for each event. The `[@metadata]` fields will not +be sent to Elasticsearch. + +Example: +[source,ruby] + filter { + if [log_type] in [ "test", "staging" ] { + mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } } + } else if [log_type] == "production" { + mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } } + } else { + mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } } + } + } + output { + elasticsearch { + index => "%{[@metadata][target_index]}" + } + } + + +==== Retry policy + +The retry policy has changed significantly in the 8.1.1 release. +This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience +either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP +request are handled differently than error codes for individual documents. + + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. + +The following document errors are handled as follows: + + * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<{version}-plugins-{type}s-{plugin}-dlq-policy>> for more info. + * 409 errors (conflict) are logged as a warning and dropped. + +Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. +It is more performant for Elasticsearch to retry these exceptions than this plugin. + +[id="{version}-plugins-{type}s-{plugin}-dlq-policy"] +==== DLQ policy + +Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately +mapping errors cannot be handled without human intervention and without looking +at the field that caused the mapping mismatch. If the DLQ is enabled, the +original events causing the mapping errors are stored in a file that can be +processed at a later time. Often times, the offending field can be removed and +re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error +happens, the problem is logged as a warning, and the event is dropped. +Check out https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html[Dead letter queues (DLQ)] for more information about processing events in the DLQ. +The list of error codes accepted for DLQ could be customized with <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> +but should be used only in motivated cases. + +[id="{version}-plugins-{type}s-{plugin}-ilm"] +==== {ilm-cap} ({ilm-init}) + +[NOTE] +-- +* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. +Any {ilm-init} settings in your plugin configuration are ignored and may cause errors. +* The {ilm-init} feature requires plugin version `9.3.1` or higher. +* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license +-- + +{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time. + +The use of {ilm} is controlled by the `ilm_enabled` +setting. By default, this setting detects whether the Elasticsearch instance +supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to +`true` or `false` to override the automatic detection, or disable {ilm-init}. + +This will overwrite the index settings and adjust the {ls} template to write +the necessary settings for the template to support {ilm}, +including the index policy and rollover alias to be used. + +{ls} creates a rollover alias for the indices to be written to, +including a pattern for how the actual indices will be named, and unless an ILM +policy that already exists has been specified, a default policy will also be +created. The default policy is configured to rollover an index when it reaches +either 50 gigabytes in size, or is 30 days old, whichever happens first. + +The default rollover alias is called `logstash`, with a default pattern for the +rollover index of `{now/d}-00001`, which will name indices on the date that the +index is rolled over, followed by an incrementing number. Note that the pattern +must end with a dash and a number that will be incremented. + +See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API documentation] for more details on naming. + +The rollover alias, ilm pattern and policy can be modified. + +See config below for an example: +[source,ruby] + output { + elasticsearch { + ilm_rollover_alias => "custom" + ilm_pattern => "000001" + ilm_policy => "custom_policy" + } + } + +[NOTE] +-- +* Custom ILM policies must already exist on the {es} cluster before they can be used. +* If the rollover alias or pattern is modified, the index template will need to be +overwritten as the settings `index.lifecycle.name` and +`index.lifecycle.rollover_alias` are automatically written to the template +* If the index property is supplied in the output definition, it will be overwritten by the rollover alias. +-- + +==== Batch Sizes + +This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es} +Bulk API] as a single request. However, if a batch exceeds 20MB we break it up +into multiple bulk requests. If a single document exceeds 20MB it is sent as a +single request. + +==== DNS Caching + +This plugin uses the JVM to lookup DNS entries and is subject to the value of +https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], +a global setting for the JVM. + +As an example, to set your DNS TTL to 1 second you would set +the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. + +Keep in mind that a connection with keepalive enabled will +not reevaluate its DNS value while the keepalive is in effect. + +==== HTTP Compression + +This plugin always reads compressed responses from {es}. +By default, it sends compressed bulk requests to {es}. + +If you are concerned about bandwidth, you can set a higher <<{version}-plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO. + +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using one of the +`user`/`password`, `cloud_auth` or `api_key` options. + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at +index level and `monitoring` permissions at cluster level. The `monitoring` +permission at cluster level is necessary to perform periodic connectivity +checks. + +[id="{version}-plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Output Configuration Options + +This plugin supports these configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-action>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_level>> |{logstash-ref}/configuration-file-structure.html#number[number], one of `[0 ~ 9]`|No +| <<{version}-plugins-{type}s-{plugin}-custom_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_auto_routing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_namespace>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_sync_fields>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-data_stream_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_custom_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-doc_as_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-document_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-document_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-healthcheck_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_enabled>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["true", "false", "auto"]`|No +| <<{version}-plugins-{type}s-{plugin}-ilm_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_policy>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-manage_template>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-parent>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-resurrect_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_initial_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_max_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_conflict>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-routing>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_lang>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-script_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["inline", "indexed", "file"]`|No +| <<{version}-plugins-{type}s-{plugin}-script_var_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-scripted_upsert>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sniffing_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-template>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-template_api>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "legacy", "composable"]`|No +| <<{version}-plugins-{type}s-{plugin}-template_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-template_overwrite>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upsert>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-version_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["internal", "external", "external_gt", "external_gte", "force"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-action"] +===== `action` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `create` for data streams, and `index` for non-time series data. + +The Elasticsearch action to perform. Valid actions are: + +- `index`: indexes a document (an event from Logstash). +- `delete`: deletes a document by id (An id is required for this action) +- `create`: indexes a document, fails if a document by that id already exists in the index. +- `update`: updates a document by id. Update has a special case where you can upsert -- update a + document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported + in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! +- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` + would use the foo field for the action. + If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}. + Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped. + +For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation]. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <<{version}-plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<{version}-plugins-{type}s-{plugin}-hosts>>, or by setting <<{version}-plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-bulk_path"] +===== `bulk_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status` + +HTTP Path to perform the _bulk requests to +* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status` +* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used. + + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `user`/`password` pair. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-compression_level"] +===== `compression_level` + + * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9` + * Default value is `1` + +The gzip compression level. Setting this value to `0` disables compression. +The compression level must be in the range of `1` (best speed) to `9` (best compression). + +Increasing the compression level will reduce the network usage but will increase the CPU usage. + +[id="{version}-plugins-{type}s-{plugin}-data_stream"] +===== `data_stream` + +* Value can be any of: `true`, `false` and `auto` +* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0. + +Defines whether data will be indexed into an Elasticsearch data stream. +The other `data_stream_*` settings will be used only if this setting is enabled. + +Logstash handles the output as a data stream when the supplied configuration +is compatible with data streams and this value is set to `auto`. +Note that <<{version}-plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility>> must be enabled (set to `v1` or `v8`) for data streams to work properly. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_auto_routing"] +===== `data_stream_auto_routing` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +Automatically routes events by deriving the data stream name using specific event +fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format. + +If enabled, the `data_stream.*` event fields will take precedence over the +`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings, +but will fall back to them if any of the fields are missing from the event. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_dataset"] +===== `data_stream_dataset` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `generic`. + +The data stream dataset used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_namespace"] +===== `data_stream_namespace` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `default`. + +The data stream namespace used to construct the data stream at index time. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_sync_fields"] +===== `data_stream_sync_fields` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Automatically adds and syncs the `data_stream.*` event fields if they are missing from the +event. This ensures that fields match the name of the data stream that is receiving events. + +NOTE: If existing `data_stream.*` event fields do not match the data stream name +and `data_stream_auto_routing` is disabled, the event fields will be +overwritten with a warning. + +[id="{version}-plugins-{type}s-{plugin}-data_stream_type"] +===== `data_stream_type` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Default value is `logs`. + +The data stream type used to construct the data stream at index time. +Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported. + +[id="{version}-plugins-{type}s-{plugin}-dlq_custom_codes"] +===== `dlq_custom_codes` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `[]`. + +List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue. +This list is an addition to the ordinary error codes considered for this feature, 400 and 404. +It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict. +The option accepts a list of natural numbers corresponding to HTTP errors codes. + +[id="{version}-plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"] +===== `dlq_on_failed_indexname_interpolation` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true`. + +If enabled, failed index name interpolation events go into dead letter queue. + +[id="{version}-plugins-{type}s-{plugin}-doc_as_upsert"] +===== `doc_as_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable `doc_as_upsert` for update mode. +Create a new document with source if `document_id` doesn't exist in Elasticsearch. + +[id="{version}-plugins-{type}s-{plugin}-document_id"] +===== `document_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The document ID for the index. Useful for overwriting existing entries in +Elasticsearch with the same ID. + +[id="{version}-plugins-{type}s-{plugin}-document_type"] +===== `document_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + * This option is deprecated + +NOTE: This option is deprecated due to the +https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal +of types in Elasticsearch 6.0]. + +NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. + +This sets the document type to write events to. Generally you should try to write only +similar events to the same 'type'. String expansion `%{foo}` works here. +If you don't set a value for this option: + +- for elasticsearch clusters 8.x: no value will be used; +- for elasticsearch clusters 7.x: the value of '_doc' will be used. + +[id="{version}-plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`,`v8`: Elastic Common Schema-compliant behavior +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema +(ECS)], including the installation of ECS-compatible index templates. The value +of this setting affects the _default_ values of: + +* <<{version}-plugins-{type}s-{plugin}-index>> +* <<{version}-plugins-{type}s-{plugin}-template_name>> +* <<{version}-plugins-{type}s-{plugin}-ilm_rollover_alias>> + +[id="{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist"] +===== `failure_type_logging_whitelist` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +NOTE: Deprecated, refer to <<{version}-plugins-{type}s-{plugin}-silence_errors_in_log>>. + +[id="{version}-plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the headers sent in each request to +an elasticsearch node. The headers will be used for any kind of request +(_bulk request, template installation, health checks and sniffing). +These custom headers will be overidden by settings like `compression_level`. + +[id="{version}-plugins-{type}s-{plugin}-healthcheck_path"] +===== `healthcheck_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path where a HEAD request is sent when a backend is marked down +the request is sent in the background to see if it has come back again +before it is once again eligible to service requests. +If you have custom firewall rules you may need to change this + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * Default value is `[//127.0.0.1]` + +Sets the host(s) of the remote instance. If given an array it will load balance +requests across the hosts specified in the `hosts` parameter. Remember the +`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg. +9200, not 9300). + +Examples: + + `"127.0.0.1"` + `["127.0.0.1:9200","127.0.0.2:9200"]` + `["http://127.0.0.1"]` + `["https://127.0.0.1:9200"]` + `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) + +Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to +prevent Logstash from sending bulk requests to the master nodes. This parameter +should reference only data or client nodes in Elasticsearch. + +Any special characters present in the URLs here MUST be URL escaped! This means +`#` should be put in as `%23` for instance. + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` +deprecated[11.17.0, Replaced by <<{version}-plugins-{type}s-{plugin}-compression_level>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Setting `true` enables gzip compression level 1 on requests. + +This setting allows you to reduce this plugin's outbound network traffic by +compressing each bulk _request_ to {es}. + +NOTE: This output plugin reads compressed _responses_ from {es} regardless + of the value of this setting. + +[id="{version}-plugins-{type}s-{plugin}-ilm_enabled"] +===== `ilm_enabled` + + * Value can be any of: `true`, `false`, `auto` + * Default value is `auto` + +The default setting of `auto` will automatically enable +{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the +Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with +the ILM feature enabled, and disable it otherwise. + +Setting this flag to `false` will disable the Index Lifecycle Management +feature, even if the Elasticsearch cluster supports ILM. +Setting this flag to `true` will enable Index Lifecycle Management feature, if +the Elasticsearch cluster supports it. This is required to enable Index +Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`. + +NOTE: This feature requires a Basic License or above to be installed on an +Elasticsearch cluster version 6.6.0 or later. + +[id="{version}-plugins-{type}s-{plugin}-ilm_pattern"] +===== `ilm_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `{now/d}-000001` + +Pattern used for generating indices managed by +{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value +specified in the pattern will be appended to the write alias, and incremented +automatically when a new index is created by ILM. + +Date Math can be used when specifying an ilm pattern, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +API docs] for details. + +NOTE: Updating the pattern will require the index template to be rewritten. + +NOTE: The pattern must finish with a dash and a number that will be automatically +incremented when indices rollover. + +NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name. +Example: 000001. See +{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path +parameters API docs] for details. + +[id="{version}-plugins-{type}s-{plugin}-ilm_policy"] +===== `ilm_policy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `logstash-policy` + +Modify this setting to use a custom Index Lifecycle Management policy, rather +than the default. If this value is not set, the default policy will be +automatically installed into Elasticsearch + +NOTE: If this setting is specified, the policy must already exist in Elasticsearch +cluster. + +[id="{version}-plugins-{type}s-{plugin}-ilm_rollover_alias"] +===== `ilm_rollover_alias` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + +The rollover alias is the alias where indices managed using Index Lifecycle +Management will be written to. + +NOTE: If both `index` and `ilm_rollover_alias` are specified, +`ilm_rollover_alias` takes precedence. + +NOTE: Updating the rollover alias will require the index template to be +rewritten. + +NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as +`index` does. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` + +The indexing target to write events to. +Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. +This can be dynamic using the `%{foo}` syntax. +The default value will partition your indices by day so you can more easily +delete old data or only search specific date ranges. +Indexes may not contain uppercase characters. +For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. +Logstash uses +http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda +formats] and the `@timestamp` field of each event is being used as source for the date. + +[id="{version}-plugins-{type}s-{plugin}-manage_template"] +===== `manage_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` for non-time series data, and `false` for data streams. + +From Logstash 1.3 onwards, a template is applied to Elasticsearch during +Logstash's startup if one with the name <<{version}-plugins-{type}s-{plugin}-template_name>> +does not already exist. +By default, the contents of this template is the default template for +`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern +`logstash-*`. Should you require support for other index names, or would like +to change the mappings in the template in general, a custom template can be +specified by setting `template` to the path of a template file. + +Setting `manage_template` to false disables this feature. If you require more +control over template creation, (e.g. creating indices dynamically based on +field names) you should set `manage_template` to false and use the REST +API to apply your templates manually. + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Pass a set of key value pairs as the URL query string. This query string is added +to every host listed in the 'hosts' configuration. If the 'hosts' list contains +urls that already have query strings, the one specified here will be appended. + +[id="{version}-plugins-{type}s-{plugin}-parent"] +===== `parent` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +For child documents, ID of the associated parent. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-path"] +===== `path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path at which the Elasticsearch server lives. Use this if you must run +Elasticsearch behind a proxy that remaps the root path for the Elasticsearch +HTTP API lives. +Note that if you use paths as components of URLs in the 'hosts' field you may +not also set this field. That will raise an error at startup + +[id="{version}-plugins-{type}s-{plugin}-pipeline"] +===== `pipeline` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value. + +Set which ingest pipeline you wish to execute for an event. You can also use +event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`. +The pipeline parameter won't be set if the value resolves to empty string (""). + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1000` + +While the output tries to reuse connections efficiently we have a maximum. +This sets the maximum number of open connections the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` + +While the output tries to reuse connections efficiently we have a maximum per endpoint. +This sets the maximum number of open connections per endpoint the output will create. +Setting this too low may mean frequently closing / opening connections +which is bad. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] + * There is no default value for this setting. + +Set the address of a forward HTTP proxy. +This setting accepts only URI arguments to prevent leaking credentials. +An empty string is treated as if proxy was not set. This is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-resurrect_delay"] +===== `resurrect_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How frequently, in seconds, to wait between resurrection attempts. +Resurrection is the process by which backend endpoints marked 'down' are checked +to see if they have come back to life + +[id="{version}-plugins-{type}s-{plugin}-retry_initial_interval"] +===== `retry_initial_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Set initial interval in seconds between bulk retries. Doubled on each retry up +to `retry_max_interval` + +[id="{version}-plugins-{type}s-{plugin}-retry_max_interval"] +===== `retry_max_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `64` + +Set max interval in seconds between bulk retries. + +[id="{version}-plugins-{type}s-{plugin}-retry_on_conflict"] +===== `retry_on_conflict` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +The number of times Elasticsearch should internally retry an update/upserted document. + +[id="{version}-plugins-{type}s-{plugin}-routing"] +===== `routing` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A routing override to be applied to all processed events. +This can be dynamic using the `%{foo}` syntax. + +[id="{version}-plugins-{type}s-{plugin}-script"] +===== `script` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set script name for scripted update mode + +Example: +[source,ruby] + output { + elasticsearch { + script => "ctx._source.message = params.event.get('message')" + } + } + +[id="{version}-plugins-{type}s-{plugin}-script_lang"] +===== `script_lang` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"painless"` + +Set the language of the used script. +When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set +this parameter to `""` (empty string). + +[id="{version}-plugins-{type}s-{plugin}-script_type"] +===== `script_type` + + * Value can be any of: `inline`, `indexed`, `file` + * Default value is `["inline"]` + +Define the type of script referenced by "script" variable + inline : "script" contains inline script + indexed : "script" contains the name of script directly indexed in elasticsearch + file : "script" contains the name of script stored in elasticsearch's config directory + +[id="{version}-plugins-{type}s-{plugin}-script_var_name"] +===== `script_var_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"event"` + +Set variable name passed to script (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-scripted_upsert"] +===== `scripted_upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +if enabled, script is in charge of creating non-existent document (scripted update) + +[id="{version}-plugins-{type}s-{plugin}-silence_errors_in_log"] +===== `silence_errors_in_log` + +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* Default value is `[]` + +Defines the list of Elasticsearch errors that you don't want to log. +A useful example is when you want to skip all 409 errors +which are `version_conflict_engine_exception`. + +[source,ruby] + output { + elasticsearch { + silence_errors_in_log => ["version_conflict_engine_exception"] + } + } + +NOTE: Deprecates <<{version}-plugins-{type}s-{plugin}-failure_type_logging_whitelist>>. + +[id="{version}-plugins-{type}s-{plugin}-sniffing"] +===== `sniffing` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +This setting asks Elasticsearch for the list of all cluster nodes and adds them +to the hosts list. + +[id="{version}-plugins-{type}s-{plugin}-sniffing_delay"] +===== `sniffing_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +How long to wait, in seconds, between sniffing attempts + +[id="{version}-plugins-{type}s-{plugin}-sniffing_path"] +===== `sniffing_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +HTTP Path to be used for the sniffing requests +the default value is computed by concatenating the path value and "_nodes/http" +if sniffing_path is set it will be used as an absolute path +do not use full URL here, only paths, e.g. "/sniff/_nodes/http" + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<{version}-plugins-{type}s-{plugin}-hosts>> or extracted from the <<{version}-plugins-{type}s-{plugin}-cloud_id>>. +If no explicit protocol is specified plain HTTP will be used. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key to use. +This key must be in the PKCS8 format and PEM encoded. +You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion. +For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is: + +[source,sh] +----- +openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key +----- + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value can be any of: `jks`, `pkcs12` +* If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + +[id="{version}-plugins-{type}s-{plugin}-template"] +===== `template` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +You can set the path to your own template here, if you so desire. +If not set, the included template will be used. + +[id="{version}-plugins-{type}s-{plugin}-template_api"] +===== `template_api` + + * Value can be any of: `auto`, `legacy`, `composable` + * Default value is `auto` + +The default setting of `auto` will use +{ref}/index-templates.html[index template API] to create index template, if the +Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher, +and use {ref}/indices-templates-v1.html[legacy template API] otherwise. + +Setting this flag to `legacy` will use legacy template API to create index template. +Setting this flag to `composable` will use index template API to create index template. + +NOTE: The format of template provided to <<{version}-plugins-{type}s-{plugin}-template>> needs to match the template API being used. + +[id="{version}-plugins-{type}s-{plugin}-template_name"] +===== `template_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value depends on whether <<{version}-plugins-{type}s-{plugin}-ecs_compatibility>> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + + +This configuration option defines how the template is named inside Elasticsearch. +Note that if you have used the template management features and subsequently +change this, you will need to prune the old template manually, e.g. + +`curl -XDELETE ` + +where `OldTemplateName` is whatever the former setting was. + +[id="{version}-plugins-{type}s-{plugin}-template_overwrite"] +===== `template_overwrite` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +The template_overwrite option will always overwrite the indicated template +in Elasticsearch with either the one indicated by template or the included one. +This option is set to false by default. If you always want to stay up to date +with the template provided by Logstash, this option could be very useful to you. +Likewise, if you have your own template file managed by puppet, for example, and +you wanted to be able to update it regularly, this option could help there as well. + +Please note that if you are using your own customized version of the Logstash +template (logstash), setting this to true will make Logstash to overwrite +the "logstash" template (i.e. removing all customized settings) + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If +a timeout occurs, the request will be retried. + +[id="{version}-plugins-{type}s-{plugin}-upsert"] +===== `upsert` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Set upsert content for update mode. +Create a new document with this parameter as json string if `document_id` doesn't exists + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Username to authenticate to a secure Elasticsearch cluster + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +How long to wait before checking for a stale connection to determine if a keepalive request is needed. +Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly. + +This client is based on Apache Commons. Here's how the +https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache +Commons documentation] describes this option: "Defines period of inactivity in +milliseconds after which persistent connections must be re-validated prior to +being leased to the consumer. Non-positive value passed to this method disables +connection validation. This check helps detect connections that have become +stale (half-closed) while kept inactive in the pool." + +[id="{version}-plugins-{type}s-{plugin}-version"] +===== `version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The version to use for indexing. Use sprintf syntax like `%{my_version}` to use +a field value here. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] for more information. + +[id="{version}-plugins-{type}s-{plugin}-version_type"] +===== `version_type` + + * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force` + * There is no default value for this setting. + +The version_type to use for indexing. See the +https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support +blog] and {ref}/docs-index_.html#_version_types[Version types] in the +Elasticsearch documentation. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options + +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert | <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| keystore | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password | <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| ssl | <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_certificate_verification | <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> +| truststore | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password | <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/google_bigquery-index.asciidoc b/docs/versioned-plugins/outputs/google_bigquery-index.asciidoc index 89e9151ea..8d8b07c76 100644 --- a/docs/versioned-plugins/outputs/google_bigquery-index.asciidoc +++ b/docs/versioned-plugins/outputs/google_bigquery-index.asciidoc @@ -5,6 +5,8 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-09-16 +| <> | 2024-01-24 | <> | 2023-08-22 | <> | 2022-11-02 | <> | 2022-09-23 @@ -22,6 +24,8 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::google_bigquery-v4.6.0.asciidoc[] +include::google_bigquery-v4.5.0.asciidoc[] include::google_bigquery-v4.4.0.asciidoc[] include::google_bigquery-v4.3.0.asciidoc[] include::google_bigquery-v4.2.0.asciidoc[] diff --git a/docs/versioned-plugins/outputs/google_bigquery-v4.5.0.asciidoc b/docs/versioned-plugins/outputs/google_bigquery-v4.5.0.asciidoc new file mode 100644 index 000000000..77c68539c --- /dev/null +++ b/docs/versioned-plugins/outputs/google_bigquery-v4.5.0.asciidoc @@ -0,0 +1,372 @@ +:plugin: google_bigquery +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.5.0 +:release_date: 2024-01-24 +:changelog_url: https://github.com/logstash-plugins/logstash-output-google_bigquery/blob/v4.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Google BigQuery output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +===== Summary + +This Logstash plugin uploads events to Google BigQuery using the streaming API +so data can become available to query nearly immediately. + +You can configure it to flush periodically, after N events or after +a certain amount of data is ingested. + +===== Environment Configuration + +You must enable BigQuery on your Google Cloud account and create a dataset to +hold the tables this plugin generates. + +You must also grant the service account this plugin uses access to the dataset. + +You can use {logstash-ref}/event-dependent-configuration.html[Logstash conditionals] +and multiple configuration blocks to upload events with different structures. + +===== Usage +This is an example of Logstash config: + +[source,ruby] +-------------------------- +output { + google_bigquery { + project_id => "folkloric-guru-278" (required) + dataset => "logs" (required) + csv_schema => "path:STRING,status:INTEGER,score:FLOAT" (required) <1> + json_key_file => "/path/to/key.json" (optional) <2> + error_directory => "/tmp/bigquery-errors" (required) + date_pattern => "%Y-%m-%dT%H:00" (optional) + flush_interval_secs => 30 (optional) + } +} +-------------------------- + +<1> Specify either a csv_schema or a json_schema. +<2> If the key is not used, then the plugin tries to find +https://cloud.google.com/docs/authentication/production[Application Default Credentials] + +===== Considerations + +* There is a small fee to insert data into BigQuery using the streaming API. +* This plugin buffers events in-memory, so make sure the flush configurations are appropriate + for your use-case and consider using + {logstash-ref}/persistent-queues.html[Logstash Persistent Queues]. +* Events will be flushed when <<{version}-plugins-{type}s-{plugin}-batch_size>>, <<{version}-plugins-{type}s-{plugin}-batch_size_bytes>>, or <<{version}-plugins-{type}s-{plugin}-flush_interval_secs>> is met, whatever comes first. + If you notice a delay in your processing or low throughput, try adjusting those settings. + +===== Additional Resources + +* https://cloud.google.com/docs/authentication/production[Application Default Credentials (ADC) Overview] +* https://cloud.google.com/bigquery/[BigQuery Introduction] +* https://cloud.google.com/bigquery/quotas[BigQuery Quotas and Limits] +* https://cloud.google.com/bigquery/docs/schemas[BigQuery Schema Formats and Types] + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Google BigQuery Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-batch_size_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-csv_schema>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-date_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-deleter_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-error_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-flush_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ignore_unknown_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-json_key_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-json_schema>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-key_password>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-key_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|*Obsolete* +| <<{version}-plugins-{type}s-{plugin}-project_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-service_account>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-skip_invalid_rows>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-table_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-table_separator>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-temp_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-temp_file_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-uploader_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + +added[4.0.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `128` + +The maximum number of messages to upload at a single time. +This number must be < 10,000. +Batching can increase performance and throughput to a point, but at the cost of per-request latency. +Too few rows per request and the overhead of each request can make ingestion inefficient. +Too many rows per request and the throughput may drop. +BigQuery recommends using about 500 rows per request, but experimentation with representative data (schema and data sizes) will help you determine the ideal batch size. + +[id="{version}-plugins-{type}s-{plugin}-batch_size_bytes"] +===== `batch_size_bytes` + +added[4.0.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1_000_000` + +An approximate number of bytes to upload as part of a batch. +This number should be < 10MB or inserts may fail. + +[id="{version}-plugins-{type}s-{plugin}-csv_schema"] +===== `csv_schema` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Schema for log data. It must follow the format `name1:type1(,name2:type2)*`. +For example, `path:STRING,status:INTEGER,score:FLOAT`. + +[id="{version}-plugins-{type}s-{plugin}-dataset"] +===== `dataset` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The BigQuery dataset the tables for the events will be added to. + +[id="{version}-plugins-{type}s-{plugin}-date_pattern"] +===== `date_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%Y-%m-%dT%H:00"` + +Time pattern for BigQuery table, defaults to hourly tables. +Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime + +[id="{version}-plugins-{type}s-{plugin}-deleter_interval_secs"] +===== `deleter_interval_secs` + +deprecated[4.0.0, Events are uploaded in real-time without being stored to disk.] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +[id="{version}-plugins-{type}s-{plugin}-error_directory"] +===== `error_directory` +added[4.0.0] + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/bigquery"`. + +The location to store events that could not be uploaded due to errors. +By default if _any_ message in an insert is invalid all will fail. +You can use <<{version}-plugins-{type}s-{plugin}-skip_invalid_rows>> to allow partial inserts. + +Consider using an additional Logstash input to pipe the contents of +these to an alert platform so you can manually fix the events. + +Or use https://cloud.google.com/storage/docs/gcs-fuse[GCS FUSE] to +transparently upload to a GCS bucket. + +Files names follow the pattern `[table name]-[UNIX timestamp].log` + +[id="{version}-plugins-{type}s-{plugin}-flush_interval_secs"] +===== `flush_interval_secs` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Uploads all data this often even if other upload criteria aren't met. + + +[id="{version}-plugins-{type}s-{plugin}-ignore_unknown_values"] +===== `ignore_unknown_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Indicates if BigQuery should ignore values that are not represented in the table schema. +If true, the extra values are discarded. +If false, BigQuery will reject the records with extra fields and the job will fail. +The default value is false. + +NOTE: You may want to add a Logstash filter like the following to remove common fields it adds: + +[source,ruby] +---------------------------------- +mutate { + remove_field => ["@version","@timestamp","path","host","type", "message"] +} +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-json_key_file"] +===== `json_key_file` + +added[4.0.0, "Replaces <<{version}-plugins-{type}s-{plugin}-key_password>>, <<{version}-plugins-{type}s-{plugin}-key_path>> and <<{version}-plugins-{type}s-{plugin}-service_account>>."] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +If Logstash is running within Google Compute Engine, the plugin can use +GCE's Application Default Credentials. Outside of GCE, you will need to +specify a Service Account JSON key file. + +[id="{version}-plugins-{type}s-{plugin}-json_schema"] +===== `json_schema` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `nil` + +Schema for log data as a hash. +These can include nested records, descriptions, and modes. + +Example: +[source,ruby] +-------------------------- +json_schema => { + fields => [{ + name => "endpoint" + type => "STRING" + description => "Request route" + }, { + name => "status" + type => "INTEGER" + mode => "NULLABLE" + }, { + name => "params" + type => "RECORD" + mode => "REPEATED" + fields => [{ + name => "key" + type => "STRING" + }, { + name => "value" + type => "STRING" + }] + }] +} +-------------------------- + +[id="{version}-plugins-{type}s-{plugin}-key_password"] +===== `key_password` + +deprecated[4.0.0, Replaced by `json_key_file` or by using ADC. See <<{version}-plugins-{type}s-{plugin}-json_key_file>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + + +[id="{version}-plugins-{type}s-{plugin}-key_path"] +===== `key_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +**Obsolete:** The PKCS12 key file format is no longer supported. + +Please use one of the following mechanisms: + + * https://cloud.google.com/docs/authentication/production[Application Default Credentials (ADC)], + configured via environment variables on Compute Engine, Kubernetes Engine, App Engine, or + Cloud Functions. + * A JSON authentication key file. You can generate them in the console for the service account + like you did with the `.P12` file or with the following command: + `gcloud iam service-accounts keys create key.json --iam-account my-sa-123@my-project-123.iam.gserviceaccount.com` + +[id="{version}-plugins-{type}s-{plugin}-project_id"] +===== `project_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Google Cloud Project ID (number, not Project Name!). + +[id="{version}-plugins-{type}s-{plugin}-service_account"] +===== `service_account` + +deprecated[4.0.0, Replaced by `json_key_file` or by using ADC. See <<{version}-plugins-{type}s-{plugin}-json_key_file>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +[id="{version}-plugins-{type}s-{plugin}-skip_invalid_rows"] +===== `skip_invalid_rows` + +added[4.1.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Insert all valid rows of a request, even if invalid rows exist. +The default value is false, which causes the entire request to fail if any invalid rows exist. + +[id="{version}-plugins-{type}s-{plugin}-table_prefix"] +===== `table_prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +BigQuery table ID prefix to be used when creating new tables for log data. +Table name will be `` + +[id="{version}-plugins-{type}s-{plugin}-table_separator"] +===== `table_separator` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"_"` + +BigQuery table separator to be added between the table_prefix and the +date suffix. + +[id="{version}-plugins-{type}s-{plugin}-temp_directory"] +===== `temp_directory` + +deprecated[4.0.0, Events are uploaded in real-time without being stored to disk.] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +[id="{version}-plugins-{type}s-{plugin}-temp_file_prefix"] +===== `temp_file_prefix` + +deprecated[4.0.0, Events are uploaded in real-time without being stored to disk] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +[id="{version}-plugins-{type}s-{plugin}-uploader_interval_secs"] +===== `uploader_interval_secs` + +deprecated[4.0.0, This field is no longer used] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Uploader interval when uploading new files to BigQuery. Adjust time based +on your time pattern (for example, for hourly files, this interval can be +around one hour). + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/google_bigquery-v4.6.0.asciidoc b/docs/versioned-plugins/outputs/google_bigquery-v4.6.0.asciidoc new file mode 100644 index 000000000..d8949d247 --- /dev/null +++ b/docs/versioned-plugins/outputs/google_bigquery-v4.6.0.asciidoc @@ -0,0 +1,355 @@ +:plugin: google_bigquery +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.6.0 +:release_date: 2024-09-16 +:changelog_url: https://github.com/logstash-plugins/logstash-output-google_bigquery/blob/v4.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Google BigQuery output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +===== Summary + +This Logstash plugin uploads events to Google BigQuery using the streaming API +so data can become available to query nearly immediately. + +You can configure it to flush periodically, after N events or after +a certain amount of data is ingested. + +===== Environment Configuration + +You must enable BigQuery on your Google Cloud account and create a dataset to +hold the tables this plugin generates. + +You must also grant the service account this plugin uses access to the dataset. + +You can use {logstash-ref}/event-dependent-configuration.html[Logstash conditionals] +and multiple configuration blocks to upload events with different structures. + +===== Usage +This is an example of Logstash config: + +[source,ruby] +-------------------------- +output { + google_bigquery { + project_id => "folkloric-guru-278" (required) + dataset => "logs" (required) + csv_schema => "path:STRING,status:INTEGER,score:FLOAT" (required) <1> + json_key_file => "/path/to/key.json" (optional) <2> + error_directory => "/tmp/bigquery-errors" (required) + date_pattern => "%Y-%m-%dT%H:00" (optional) + flush_interval_secs => 30 (optional) + } +} +-------------------------- + +<1> Specify either a csv_schema or a json_schema. +<2> If the key is not used, then the plugin tries to find +https://cloud.google.com/docs/authentication/production[Application Default Credentials] + +===== Considerations + +* There is a small fee to insert data into BigQuery using the streaming API. +* This plugin buffers events in-memory, so make sure the flush configurations are appropriate + for your use-case and consider using + {logstash-ref}/persistent-queues.html[Logstash Persistent Queues]. +* Events will be flushed when <<{version}-plugins-{type}s-{plugin}-batch_size>>, <<{version}-plugins-{type}s-{plugin}-batch_size_bytes>>, or <<{version}-plugins-{type}s-{plugin}-flush_interval_secs>> is met, whatever comes first. + If you notice a delay in your processing or low throughput, try adjusting those settings. + +===== Additional Resources + +* https://cloud.google.com/docs/authentication/production[Application Default Credentials (ADC) Overview] +* https://cloud.google.com/bigquery/[BigQuery Introduction] +* https://cloud.google.com/bigquery/quotas[BigQuery Quotas and Limits] +* https://cloud.google.com/bigquery/docs/schemas[BigQuery Schema Formats and Types] + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Google BigQuery Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-batch_size_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-csv_schema>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-dataset>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-date_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-deleter_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-error_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-flush_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ignore_unknown_values>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-json_key_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-json_schema>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-key_password>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-project_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-service_account>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-skip_invalid_rows>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-table_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-table_separator>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-temp_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-temp_file_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-uploader_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|__Deprecated__ +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + +added[4.0.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `128` + +The maximum number of messages to upload at a single time. +This number must be < 10,000. +Batching can increase performance and throughput to a point, but at the cost of per-request latency. +Too few rows per request and the overhead of each request can make ingestion inefficient. +Too many rows per request and the throughput may drop. +BigQuery recommends using about 500 rows per request, but experimentation with representative data (schema and data sizes) will help you determine the ideal batch size. + +[id="{version}-plugins-{type}s-{plugin}-batch_size_bytes"] +===== `batch_size_bytes` + +added[4.0.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1_000_000` + +An approximate number of bytes to upload as part of a batch. +This number should be < 10MB or inserts may fail. + +[id="{version}-plugins-{type}s-{plugin}-csv_schema"] +===== `csv_schema` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +Schema for log data. It must follow the format `name1:type1(,name2:type2)*`. +For example, `path:STRING,status:INTEGER,score:FLOAT`. + +[id="{version}-plugins-{type}s-{plugin}-dataset"] +===== `dataset` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The BigQuery dataset the tables for the events will be added to. + +[id="{version}-plugins-{type}s-{plugin}-date_pattern"] +===== `date_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%Y-%m-%dT%H:00"` + +Time pattern for BigQuery table, defaults to hourly tables. +Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime + +[id="{version}-plugins-{type}s-{plugin}-deleter_interval_secs"] +===== `deleter_interval_secs` + +deprecated[4.0.0, Events are uploaded in real-time without being stored to disk.] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + +[id="{version}-plugins-{type}s-{plugin}-error_directory"] +===== `error_directory` +added[4.0.0] + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/bigquery"`. + +The location to store events that could not be uploaded due to errors. +By default if _any_ message in an insert is invalid all will fail. +You can use <<{version}-plugins-{type}s-{plugin}-skip_invalid_rows>> to allow partial inserts. + +Consider using an additional Logstash input to pipe the contents of +these to an alert platform so you can manually fix the events. + +Or use https://cloud.google.com/storage/docs/gcs-fuse[GCS FUSE] to +transparently upload to a GCS bucket. + +Files names follow the pattern `[table name]-[UNIX timestamp].log` + +[id="{version}-plugins-{type}s-{plugin}-flush_interval_secs"] +===== `flush_interval_secs` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Uploads all data this often even if other upload criteria aren't met. + + +[id="{version}-plugins-{type}s-{plugin}-ignore_unknown_values"] +===== `ignore_unknown_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Indicates if BigQuery should ignore values that are not represented in the table schema. +If true, the extra values are discarded. +If false, BigQuery will reject the records with extra fields and the job will fail. +The default value is false. + +NOTE: You may want to add a Logstash filter like the following to remove common fields it adds: + +[source,ruby] +---------------------------------- +mutate { + remove_field => ["@version","@timestamp","path","host","type", "message"] +} +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-json_key_file"] +===== `json_key_file` + +added[4.0.0, "Replaces <<{version}-plugins-{type}s-{plugin}-key_password>> and <<{version}-plugins-{type}s-{plugin}-service_account>>."] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +If Logstash is running within Google Compute Engine, the plugin can use +GCE's Application Default Credentials. Outside of GCE, you will need to +specify a Service Account JSON key file. + +[id="{version}-plugins-{type}s-{plugin}-json_schema"] +===== `json_schema` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `nil` + +Schema for log data as a hash. +These can include nested records, descriptions, and modes. + +Example: +[source,ruby] +-------------------------- +json_schema => { + fields => [{ + name => "endpoint" + type => "STRING" + description => "Request route" + }, { + name => "status" + type => "INTEGER" + mode => "NULLABLE" + }, { + name => "params" + type => "RECORD" + mode => "REPEATED" + fields => [{ + name => "key" + type => "STRING" + }, { + name => "value" + type => "STRING" + }] + }] +} +-------------------------- + +[id="{version}-plugins-{type}s-{plugin}-key_password"] +===== `key_password` + +deprecated[4.0.0, Replaced by `json_key_file` or by using ADC. See <<{version}-plugins-{type}s-{plugin}-json_key_file>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + + +[id="{version}-plugins-{type}s-{plugin}-project_id"] +===== `project_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Google Cloud Project ID (number, not Project Name!). + +[id="{version}-plugins-{type}s-{plugin}-service_account"] +===== `service_account` + +deprecated[4.0.0, Replaced by `json_key_file` or by using ADC. See <<{version}-plugins-{type}s-{plugin}-json_key_file>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +[id="{version}-plugins-{type}s-{plugin}-skip_invalid_rows"] +===== `skip_invalid_rows` + +added[4.1.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Insert all valid rows of a request, even if invalid rows exist. +The default value is false, which causes the entire request to fail if any invalid rows exist. + +[id="{version}-plugins-{type}s-{plugin}-table_prefix"] +===== `table_prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +BigQuery table ID prefix to be used when creating new tables for log data. +Table name will be `` + +[id="{version}-plugins-{type}s-{plugin}-table_separator"] +===== `table_separator` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"_"` + +BigQuery table separator to be added between the table_prefix and the +date suffix. + +[id="{version}-plugins-{type}s-{plugin}-temp_directory"] +===== `temp_directory` + +deprecated[4.0.0, Events are uploaded in real-time without being stored to disk.] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +[id="{version}-plugins-{type}s-{plugin}-temp_file_prefix"] +===== `temp_file_prefix` + +deprecated[4.0.0, Events are uploaded in real-time without being stored to disk] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +[id="{version}-plugins-{type}s-{plugin}-uploader_interval_secs"] +===== `uploader_interval_secs` + +deprecated[4.0.0, This field is no longer used] + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Uploader interval when uploading new files to BigQuery. Adjust time based +on your time pattern (for example, for hourly files, this interval can be +around one hour). + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/google_cloud_storage-index.asciidoc b/docs/versioned-plugins/outputs/google_cloud_storage-index.asciidoc index 255f51ae7..cdb70ca4a 100644 --- a/docs/versioned-plugins/outputs/google_cloud_storage-index.asciidoc +++ b/docs/versioned-plugins/outputs/google_cloud_storage-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-09-16 | <> | 2023-08-22 | <> | 2022-11-02 | <> | 2022-09-23 @@ -20,6 +21,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::google_cloud_storage-v4.5.0.asciidoc[] include::google_cloud_storage-v4.4.0.asciidoc[] include::google_cloud_storage-v4.3.0.asciidoc[] include::google_cloud_storage-v4.2.0.asciidoc[] diff --git a/docs/versioned-plugins/outputs/google_cloud_storage-v4.5.0.asciidoc b/docs/versioned-plugins/outputs/google_cloud_storage-v4.5.0.asciidoc new file mode 100644 index 000000000..91a5641d6 --- /dev/null +++ b/docs/versioned-plugins/outputs/google_cloud_storage-v4.5.0.asciidoc @@ -0,0 +1,295 @@ +:plugin: google_cloud_storage +:type: output +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v4.5.0 +:release_date: 2024-09-16 +:changelog_url: https://github.com/logstash-plugins/logstash-output-google_cloud_storage/blob/v4.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Google Cloud Storage output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +A plugin to upload log events to Google Cloud Storage (GCS), rolling +files based on the date pattern provided as a configuration setting. Events +are written to files locally and, once file is closed, this plugin uploads +it to the configured bucket. + +For more info on Google Cloud Storage, please go to: +https://cloud.google.com/products/cloud-storage + +In order to use this plugin, a Google service account must be used. For +more information, please refer to: +https://developers.google.com/storage/docs/authentication#service_accounts + +Recommendation: experiment with the settings depending on how much log +data you generate, so the uploader can keep up with the generated logs. +Using gzip output can be a good option to reduce network traffic when +uploading the log files and in terms of storage costs as well. + +==== Usage + +This is an example of logstash config: + +[source,json] +-------------------------- +output { + google_cloud_storage { + bucket => "my_bucket" (required) + json_key_file => "/path/to/privatekey.json" (optional) + temp_directory => "/tmp/logstash-gcs" (optional) + log_file_prefix => "logstash_gcs" (optional) + max_file_size_kbytes => 1024 (optional) + output_format => "plain" (optional) + date_pattern => "%Y-%m-%dT%H:00" (optional) + flush_interval_secs => 2 (optional) + gzip => false (optional) + gzip_content_encoding => false (optional) + uploader_interval_secs => 60 (optional) + include_uuid => true (optional) + include_hostname => true (optional) + } +} +-------------------------- + +===== Additional Resources + +* https://cloud.google.com/docs/authentication/production[Application Default Credentials (ADC) Overview] +* https://cloud.google.com/storage/[Cloud Storage Introduction] +* https://cloud.google.com/storage/pricing[Pricing Information] + + +==== Improvements TODO List + +* Support logstash event variables to determine filename. +* Turn Google API code into a Plugin Mixin (like AwsConfig). +* There's no recover method, so if logstash/plugin crashes, files may not +be uploaded to GCS. +* Allow user to configure file name. +* Allow parallel uploads for heavier loads (+ connection configuration if +exposed by Ruby API client) + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Google_cloud_storage Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-date_pattern>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-flush_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-gzip>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-gzip_content_encoding>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_hostname>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-include_uuid>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-json_key_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-key_password>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-log_file_prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-max_concurrent_uploads>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_file_size_kbytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-output_format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["json", "plain", nil]`|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-service_account>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-temp_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-uploader_interval_secs>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +GCS bucket name, without "gs://" or any other prefix. + +[id="{version}-plugins-{type}s-{plugin}-date_pattern"] +===== `date_pattern` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"%Y-%m-%dT%H:00"` + +Time pattern for log file, defaults to hourly files. +Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime + +[id="{version}-plugins-{type}s-{plugin}-flush_interval_secs"] +===== `flush_interval_secs` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Flush interval in seconds for flushing writes to log files. 0 will flush +on every message. + +[id="{version}-plugins-{type}s-{plugin}-gzip"] +===== `gzip` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Gzip output stream when writing events to log files, set +`Content-Type` to `application/gzip` instead of `text/plain`, and +use file suffix `.log.gz` instead of `.log`. + +[id="{version}-plugins-{type}s-{plugin}-gzip_content_encoding"] +===== `gzip_content_encoding` + +added[3.3.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Gzip output stream when writing events to log files and set `Content-Encoding` to `gzip`. +This will upload your files as `gzip` saving network and storage costs, but they will be +transparently decompressed when you read them from the storage bucket. + +See the Cloud Storage documentation on https://cloud.google.com/storage/docs/metadata#content-encoding[metadata] and +https://cloud.google.com/storage/docs/transcoding#content-type_vs_content-encoding[transcoding] +for more information. + +**Note**: It is not recommended to use both `gzip_content_encoding` and `gzip`. +This compresses your file _twice_, will increase the work your machine does and makes +the files larger than just compressing once. + +[id="{version}-plugins-{type}s-{plugin}-include_hostname"] +===== `include_hostname` + +added[3.1.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should the hostname be included in the file name? +You may want to turn this off for privacy reasons or if you are running multiple +instances of Logstash and need to match the files you create with a simple glob +such as if you wanted to import files to BigQuery. + + +[id="{version}-plugins-{type}s-{plugin}-include_uuid"] +===== `include_uuid` + +added[3.1.0] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Adds a UUID to the end of a file name. +You may want to enable this feature so files don't clobber one another if you're +running multiple instances of Logstash or if you expect frequent node restarts. + +[id="{version}-plugins-{type}s-{plugin}-json_key_file"] +===== `json_key_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `nil` + +The plugin can use +https://cloud.google.com/docs/authentication/production[Application Default Credentials (ADC)], +if it's running on Compute Engine, Kubernetes Engine, App Engine, or Cloud Functions. + +Outside of Google Cloud, you will need create a Service Account JSON key file through the +web interface or with the following command: +`gcloud iam service-accounts keys create key.json --iam-account my-sa-123@my-project-123.iam.gserviceaccount.com` + +[id="{version}-plugins-{type}s-{plugin}-key_password"] +===== `key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"notasecret"` + +**Deprecated** this feature is no longer used, the setting is now a part of <<{version}-plugins-{type}s-{plugin}-json_key_file>>. + +[id="{version}-plugins-{type}s-{plugin}-log_file_prefix"] +===== `log_file_prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash_gcs"` + +Log file prefix. Log file will follow the format: +_hostname_date<.part?>.log + +[id="{version}-plugins-{type}s-{plugin}-max_concurrent_uploads"] +===== `max_concurrent_uploads` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Sets the maximum number of concurrent uploads to Cloud Storage at a time. +Uploads are I/O bound so it makes sense to tune this paramater with regards +to the network bandwidth available and the latency between your server and +Cloud Storage. + +[id="{version}-plugins-{type}s-{plugin}-max_file_size_kbytes"] +===== `max_file_size_kbytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` + +Sets max file size in kbytes. 0 disable max file check. + +[id="{version}-plugins-{type}s-{plugin}-output_format"] +===== `output_format` + + * Value can be any of: `json`, `plain`, or no value + * Default value is no value + +**Deprecated**, this feature will be removed in the next major release. Use codecs instead. + + * If you are using the `json` value today, switch to the `json_lines` codec. + * If you are using the `plain` value today, switch to the `line` codec. + +The event format you want to store in files. Defaults to plain text. + +Note: if you want to use a codec you MUST not set this value. + +[id="{version}-plugins-{type}s-{plugin}-service_account"] +===== `service_account` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +**Deprecated** this feature is no longer used, the setting is now a part of <<{version}-plugins-{type}s-{plugin}-json_key_file>>. + +[id="{version}-plugins-{type}s-{plugin}-temp_directory"] +===== `temp_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Directory where temporary files are stored. +Defaults to /tmp/logstash-gcs- + +[id="{version}-plugins-{type}s-{plugin}-uploader_interval_secs"] +===== `uploader_interval_secs` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +Uploader interval when uploading new files to GCS. Adjust time based +on your time pattern (for example, for hourly files, this interval can be +around one hour). + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/http-index.asciidoc b/docs/versioned-plugins/outputs/http-index.asciidoc index cfb20b852..ef41ff83c 100644 --- a/docs/versioned-plugins/outputs/http-index.asciidoc +++ b/docs/versioned-plugins/outputs/http-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-11-21 +| <> | 2024-11-18 +| <> | 2024-06-19 +| <> | 2024-06-04 | <> | 2023-09-01 | <> | 2022-03-28 | <> | 2022-03-04 @@ -28,6 +32,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-08 |======================================================================= +include::http-v6.0.0.asciidoc[] +include::http-v5.7.1.asciidoc[] +include::http-v5.7.0.asciidoc[] +include::http-v5.6.1.asciidoc[] include::http-v5.6.0.asciidoc[] include::http-v5.5.0.asciidoc[] include::http-v5.4.1.asciidoc[] diff --git a/docs/versioned-plugins/outputs/http-v5.6.1.asciidoc b/docs/versioned-plugins/outputs/http-v5.6.1.asciidoc new file mode 100644 index 000000000..0edd22538 --- /dev/null +++ b/docs/versioned-plugins/outputs/http-v5.6.1.asciidoc @@ -0,0 +1,570 @@ +:plugin: http +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.1 +:release_date: 2024-06-04 +:changelog_url: https://github.com/logstash-plugins/logstash-output-http/blob/v5.6.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output lets you send events to a generic HTTP(S) endpoint. + +This output will execute up to 'pool_max' requests in parallel for performance. +Consider this when tuning this plugin for performance. + +Additionally, note that when parallel execution is used strict ordering of events is not +guaranteed! + +Beware, this gem does not yet support codecs. Please use the 'format' option for now. + +[id="{version}-plugins-{type}s-{plugin}-retry_policy"] +==== Retry policy + +This output has two levels of retry: library and plugin. + +[id="{version}-plugins-{type}s-{plugin}-library_retry"] +===== Library retry + +The library retry applies to IO related failures. +Non retriable errors include SSL related problems, unresolvable hosts, +connection issues, and OS/JVM level interruptions happening during a request. + +The options for library retry are: + +* <<{version}-plugins-{type}s-{plugin}-automatic_retries,`automatic_retries`>>. +Controls the number of times the plugin should retry after failures at the library level. +* <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent,`retry_non_idempotent`>>. +When set to `false`, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be +retried. + +[id="{version}-plugins-{type}s-{plugin}-plugin_retry"] +===== Plugin retry + +The options for plugin level retry are: + +* <<{version}-plugins-{type}s-{plugin}-retry_failed,`retry_failed`>>. +When set to `true`, the plugin retries indefinitely for HTTP error response codes defined +in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> option +(429, 500, 502, 503, 504) and retryable exceptions (socket timeout/ error, DNS resolution failure and client protocol exception). +* <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>>. +Sets http response codes that trigger a retry. + +NOTE: The `retry_failed` option does not control the library level retry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Http Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-content_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["json", "json_batch", "form", "message"]`|No +| <<{version}-plugins-{type}s-{plugin}-headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-http_method>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["put", "post", "patch", "delete", "get", "head"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-ignorable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-mapping>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_failed>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retryable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We recommend setting this option +to a value other than zero if the <<{version}-plugins-{type}s-{plugin}-keepalive,`keepalive` option>> is enabled. +Some servers incorrectly end keepalives early, requiring a retry. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="{version}-plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="{version}-plugins-{type}s-{plugin}-client_key"] +===== `client_key` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_key>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-content_type"] +===== `content_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Content type + +If not specified, this defaults to the following: + +* if format is "json", "application/json" +* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request. +* if format is "form", "application/x-www-form-urlencoded" + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-format"] +===== `format` + + * Value can be any of: `json`, `json_batch`, `form`, `message` + * Default value is `"json"` + +Set the format of the http body. + +If json_batch, each batch of events received by this output will be placed +into a single JSON array and sent in one request. This is particularly useful +for high throughput scenarios such as sending data between Logstash instaces. + +If form, then the body will be the mapping (or whole event) converted +into a query parameter string, e.g. `foo=bar&baz=fizz...` + +If message, then the body will be the result of formatting the event according to message + +Otherwise, the event is sent as json. + +[id="{version}-plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Custom headers to use +format is `headers => ["X-My-Header", "%{host}"]` + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable request compression support. With this enabled the plugin will compress +http requests using gzip. + +[id="{version}-plugins-{type}s-{plugin}-http_method"] +===== `http_method` + + * This is a required setting. + * Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head` + * There is no default value for this setting. + +The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head" + +[id="{version}-plugins-{type}s-{plugin}-ignorable_codes"] +===== `ignorable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +If you would like to consider some non-2xx codes to be successes +enumerate them here. Responses returning these codes will be considered successes + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-mapping"] +===== `mapping` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +This lets you choose the structure and parts of the event that are sent. + + +For example: +[source,ruby] + mapping => {"foo" => "%{host}" + "bar" => "%{type}"} + +[id="{version}-plugins-{type}s-{plugin}-message"] +===== `message` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + + + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +This module makes it easy to add a very fully configured HTTP client to logstash +based on [Manticore](https://github.com/cheald/manticore). +For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller +Timeout (in seconds) for the entire request + +[id="{version}-plugins-{type}s-{plugin}-retry_failed"] +===== `retry_failed` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Note that this option controls plugin-level retries only. +It has no affect on library-level retries. + +Set this option to `false` if you want to disable infinite retries for HTTP error response codes defined in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> or +retryable exceptions (Timeout, SocketException, ClientProtocolException, ResolutionFailure and SocketTimeout). +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When this option is set to `false` and `automatic_retries` is enabled, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +When set to `true` and `automatic_retries` is enabled, this will cause non-idempotent HTTP verbs (such as POST) to be retried. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retryable_codes"] +===== `retryable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `[429, 500, 502, 503, 504]` + +If the plugin encounters these response codes, the plugin will retry indefinitely. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URL to use + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking if the connection is stale before executing a request on a connection using keepalive. +You may want to set this lower, possibly to 0 if you get connection errors regularly +Quoting the Apache commons docs (this client is based Apache Commmons): +'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.' +See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info] + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/http-v5.7.0.asciidoc b/docs/versioned-plugins/outputs/http-v5.7.0.asciidoc new file mode 100644 index 000000000..ef9e16b92 --- /dev/null +++ b/docs/versioned-plugins/outputs/http-v5.7.0.asciidoc @@ -0,0 +1,580 @@ +:plugin: http +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.7.0 +:release_date: 2024-06-19 +:changelog_url: https://github.com/logstash-plugins/logstash-output-http/blob/v5.7.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output lets you send events to a generic HTTP(S) endpoint. + +This output will execute up to 'pool_max' requests in parallel for performance. +Consider this when tuning this plugin for performance. + +Additionally, note that when parallel execution is used strict ordering of events is not +guaranteed! + +Beware, this gem does not yet support codecs. Please use the 'format' option for now. + +[id="{version}-plugins-{type}s-{plugin}-retry_policy"] +==== Retry policy + +This output has two levels of retry: library and plugin. + +[id="{version}-plugins-{type}s-{plugin}-library_retry"] +===== Library retry + +The library retry applies to IO related failures. +Non retriable errors include SSL related problems, unresolvable hosts, +connection issues, and OS/JVM level interruptions happening during a request. + +The options for library retry are: + +* <<{version}-plugins-{type}s-{plugin}-automatic_retries,`automatic_retries`>>. +Controls the number of times the plugin should retry after failures at the library level. +* <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent,`retry_non_idempotent`>>. +When set to `false`, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be +retried. + +[id="{version}-plugins-{type}s-{plugin}-plugin_retry"] +===== Plugin retry + +The options for plugin level retry are: + +* <<{version}-plugins-{type}s-{plugin}-retry_failed,`retry_failed`>>. +When set to `true`, the plugin retries indefinitely for HTTP error response codes defined +in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> option +(429, 500, 502, 503, 504) and retryable exceptions (socket timeout/ error, DNS resolution failure and client protocol exception). +* <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>>. +Sets http response codes that trigger a retry. + +NOTE: The `retry_failed` option does not control the library level retry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Http Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-content_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["json", "json_batch", "form", "message"]`|No +| <<{version}-plugins-{type}s-{plugin}-headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-http_method>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["put", "post", "patch", "delete", "get", "head"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-ignorable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-mapping>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_failed>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retryable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We recommend setting this option +to a value other than zero if the <<{version}-plugins-{type}s-{plugin}-keepalive,`keepalive` option>> is enabled. +Some servers incorrectly end keepalives early, requiring a retry. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="{version}-plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="{version}-plugins-{type}s-{plugin}-client_key"] +===== `client_key` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_key>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-content_type"] +===== `content_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Content type + +If not specified, this defaults to the following: + +* if format is "json", "application/json" +* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request. +* if format is "form", "application/x-www-form-urlencoded" + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-format"] +===== `format` + + * Value can be any of: `json`, `json_batch`, `form`, `message` + * Default value is `"json"` + +Set the format of the http body. + +If json_batch, each batch of events received by this output will be placed +into a single JSON array and sent in one request. This is particularly useful +for high throughput scenarios such as sending data between Logstash instaces. + +If form, then the body will be the mapping (or whole event) converted +into a query parameter string, e.g. `foo=bar&baz=fizz...` + +If message, then the body will be the result of formatting the event according to message + +Otherwise, the event is sent as json. + +[id="{version}-plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Custom headers to use +format is `headers => ["X-My-Header", "%{host}"]` + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable request compression support. With this enabled the plugin will compress +http requests using gzip. + +[id="{version}-plugins-{type}s-{plugin}-http_method"] +===== `http_method` + + * This is a required setting. + * Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head` + * There is no default value for this setting. + +The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head" + +[id="{version}-plugins-{type}s-{plugin}-ignorable_codes"] +===== `ignorable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +If you would like to consider some non-2xx codes to be successes +enumerate them here. Responses returning these codes will be considered successes + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-mapping"] +===== `mapping` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +This lets you choose the structure and parts of the event that are sent. + + +For example: +[source,ruby] + mapping => {"foo" => "%{host}" + "bar" => "%{type}"} + +[id="{version}-plugins-{type}s-{plugin}-message"] +===== `message` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + + + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +This module makes it easy to add a very fully configured HTTP client to logstash +based on [Manticore](https://github.com/cheald/manticore). +For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller +Timeout (in seconds) for the entire request + +[id="{version}-plugins-{type}s-{plugin}-retry_failed"] +===== `retry_failed` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Note that this option controls plugin-level retries only. +It has no affect on library-level retries. + +Set this option to `false` if you want to disable infinite retries for HTTP error response codes defined in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> or +retryable exceptions (Timeout, SocketException, ClientProtocolException, ResolutionFailure and SocketTimeout). +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When this option is set to `false` and `automatic_retries` is enabled, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +When set to `true` and `automatic_retries` is enabled, this will cause non-idempotent HTTP verbs (such as POST) to be retried. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retryable_codes"] +===== `retryable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `[429, 500, 502, 503, 504]` + +If the plugin encounters these response codes, the plugin will retry indefinitely. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URL to use + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking if the connection is stale before executing a request on a connection using keepalive. +You may want to set this lower, possibly to 0 if you get connection errors regularly +Quoting the Apache commons docs (this client is based Apache Commmons): +'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.' +See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info] + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/http-v5.7.1.asciidoc b/docs/versioned-plugins/outputs/http-v5.7.1.asciidoc new file mode 100644 index 000000000..1b2064904 --- /dev/null +++ b/docs/versioned-plugins/outputs/http-v5.7.1.asciidoc @@ -0,0 +1,580 @@ +:plugin: http +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.7.1 +:release_date: 2024-11-18 +:changelog_url: https://github.com/logstash-plugins/logstash-output-http/blob/v5.7.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output lets you send events to a generic HTTP(S) endpoint. + +This output will execute up to 'pool_max' requests in parallel for performance. +Consider this when tuning this plugin for performance. + +Additionally, note that when parallel execution is used strict ordering of events is not +guaranteed! + +Beware, this gem does not yet support codecs. Please use the 'format' option for now. + +[id="{version}-plugins-{type}s-{plugin}-retry_policy"] +==== Retry policy + +This output has two levels of retry: library and plugin. + +[id="{version}-plugins-{type}s-{plugin}-library_retry"] +===== Library retry + +The library retry applies to IO related failures. +Non retriable errors include SSL related problems, unresolvable hosts, +connection issues, and OS/JVM level interruptions happening during a request. + +The options for library retry are: + +* <<{version}-plugins-{type}s-{plugin}-automatic_retries,`automatic_retries`>>. +Controls the number of times the plugin should retry after failures at the library level. +* <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent,`retry_non_idempotent`>>. +When set to `false`, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be +retried. + +[id="{version}-plugins-{type}s-{plugin}-plugin_retry"] +===== Plugin retry + +The options for plugin level retry are: + +* <<{version}-plugins-{type}s-{plugin}-retry_failed,`retry_failed`>>. +When set to `true`, the plugin retries indefinitely for HTTP error response codes defined +in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> option +(429, 500, 502, 503, 504) and retryable exceptions (socket timeout/ error, DNS resolution failure and client protocol exception). +* <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>>. +Sets http response codes that trigger a retry. + +NOTE: The `retry_failed` option does not control the library level retry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Http Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-content_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["json", "json_batch", "form", "message"]`|No +| <<{version}-plugins-{type}s-{plugin}-headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-http_method>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["put", "post", "patch", "delete", "get", "head"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-ignorable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-mapping>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_failed>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retryable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We recommend setting this option +to a value other than zero if the <<{version}-plugins-{type}s-{plugin}-keepalive,`keepalive` option>> is enabled. +Some servers incorrectly end keepalives early, requiring a retry. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-cacert"] +===== `cacert` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="{version}-plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="{version}-plugins-{type}s-{plugin}-client_key"] +===== `client_key` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_key>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-content_type"] +===== `content_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Content type + +If not specified, this defaults to the following: + +* if format is "json", "application/json" +* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request. +* if format is "form", "application/x-www-form-urlencoded" + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-format"] +===== `format` + + * Value can be any of: `json`, `json_batch`, `form`, `message` + * Default value is `"json"` + +Set the format of the http body. + +If json_batch, each batch of events received by this output will be placed +into a single JSON array and sent in one request. This is particularly useful +for high throughput scenarios such as sending data between Logstash instaces. + +If form, then the body will be the mapping (or whole event) converted +into a query parameter string, e.g. `foo=bar&baz=fizz...` + +If message, then the body will be the result of formatting the event according to message + +Otherwise, the event is sent as json. + +[id="{version}-plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Custom headers to use +format is `headers => ["X-My-Header", "%{host}"]` + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable request compression support. With this enabled the plugin will compress +http requests using gzip. + +[id="{version}-plugins-{type}s-{plugin}-http_method"] +===== `http_method` + + * This is a required setting. + * Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head` + * There is no default value for this setting. + +The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head" + +[id="{version}-plugins-{type}s-{plugin}-ignorable_codes"] +===== `ignorable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +If you would like to consider some non-2xx codes to be successes +enumerate them here. Responses returning these codes will be considered successes + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-mapping"] +===== `mapping` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +This lets you choose the structure and parts of the event that are sent. + + +For example: +[source,ruby] + mapping => {"foo" => "%{host}" + "bar" => "%{type}"} + +[id="{version}-plugins-{type}s-{plugin}-message"] +===== `message` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + + + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +This module makes it easy to add a very fully configured HTTP client to logstash +based on [Manticore](https://github.com/cheald/manticore). +For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller +Timeout (in seconds) for the entire request + +[id="{version}-plugins-{type}s-{plugin}-retry_failed"] +===== `retry_failed` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Note that this option controls plugin-level retries only. +It has no affect on library-level retries. + +Set this option to `false` if you want to disable infinite retries for HTTP error response codes defined in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> or +retryable exceptions (Timeout, SocketException, ClientProtocolException, ResolutionFailure and SocketTimeout). +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When this option is set to `false` and `automatic_retries` is enabled, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +When set to `true` and `automatic_retries` is enabled, this will cause non-idempotent HTTP verbs (such as POST) to be retried. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retryable_codes"] +===== `retryable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `[429, 500, 502, 503, 504]` + +If the plugin encounters these response codes, the plugin will retry indefinitely. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-truststore"] +===== `truststore` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="{version}-plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="{version}-plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` +deprecated[5.6.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URL to use + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking if the connection is stale before executing a request on a connection using keepalive. +You may want to set this lower, possibly to 0 if you get connection errors regularly +Quoting the Apache commons docs (this client is based Apache Commmons): +'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.' +See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info] + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/http-v6.0.0.asciidoc b/docs/versioned-plugins/outputs/http-v6.0.0.asciidoc new file mode 100644 index 000000000..45f8ddaaa --- /dev/null +++ b/docs/versioned-plugins/outputs/http-v6.0.0.asciidoc @@ -0,0 +1,513 @@ +:plugin: http +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.0.0 +:release_date: 2024-11-21 +:changelog_url: https://github.com/logstash-plugins/logstash-output-http/blob/v6.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Http output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output lets you send events to a generic HTTP(S) endpoint. + +This output will execute up to 'pool_max' requests in parallel for performance. +Consider this when tuning this plugin for performance. + +Additionally, note that when parallel execution is used strict ordering of events is not +guaranteed! + +Beware, this gem does not yet support codecs. Please use the 'format' option for now. + +[id="{version}-plugins-{type}s-{plugin}-retry_policy"] +==== Retry policy + +This output has two levels of retry: library and plugin. + +[id="{version}-plugins-{type}s-{plugin}-library_retry"] +===== Library retry + +The library retry applies to IO related failures. +Non retriable errors include SSL related problems, unresolvable hosts, +connection issues, and OS/JVM level interruptions happening during a request. + +The options for library retry are: + +* <<{version}-plugins-{type}s-{plugin}-automatic_retries,`automatic_retries`>>. +Controls the number of times the plugin should retry after failures at the library level. +* <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent,`retry_non_idempotent`>>. +When set to `false`, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be +retried. + +[id="{version}-plugins-{type}s-{plugin}-plugin_retry"] +===== Plugin retry + +The options for plugin level retry are: + +* <<{version}-plugins-{type}s-{plugin}-retry_failed,`retry_failed`>>. +When set to `true`, the plugin retries indefinitely for HTTP error response codes defined +in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> option +(429, 500, 502, 503, 504) and retryable exceptions (socket timeout/ error, DNS resolution failure and client protocol exception). +* <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>>. +Sets http response codes that trigger a retry. + +NOTE: The `retry_failed` option does not control the library level retry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Http Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `6.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. +Please check out <<{version}-plugins-{type}s-{plugin}-obsolete-options>> for details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-automatic_retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connect_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-content_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cookies>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-follow_redirects>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-format>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["json", "json_batch", "form", "message"]`|No +| <<{version}-plugins-{type}s-{plugin}-headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-http_compression>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-http_method>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["put", "post", "patch", "delete", "get", "head"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-ignorable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-keepalive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-mapping>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-pool_max_per_route>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |<<,>>|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_failed>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_non_idempotent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retryable_codes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-socket_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-url>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-validate_after_inactivity>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How many times should the client retry a failing URL. We recommend setting this option +to a value other than zero if the <<{version}-plugins-{type}s-{plugin}-keepalive,`keepalive` option>> is enabled. +Some servers incorrectly end keepalives early, requiring a retry. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + + +[id="{version}-plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-content_type"] +===== `content_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Content type + +If not specified, this defaults to the following: + +* if format is "json", "application/json" +* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request. +* if format is "form", "application/x-www-form-urlencoded" + +[id="{version}-plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="{version}-plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="{version}-plugins-{type}s-{plugin}-format"] +===== `format` + + * Value can be any of: `json`, `json_batch`, `form`, `message` + * Default value is `"json"` + +Set the format of the http body. + +If json_batch, each batch of events received by this output will be placed +into a single JSON array and sent in one request. This is particularly useful +for high throughput scenarios such as sending data between Logstash instaces. + +If form, then the body will be the mapping (or whole event) converted +into a query parameter string, e.g. `foo=bar&baz=fizz...` + +If message, then the body will be the result of formatting the event according to message + +Otherwise, the event is sent as json. + +[id="{version}-plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +Custom headers to use +format is `headers => ["X-My-Header", "%{host}"]` + +[id="{version}-plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable request compression support. With this enabled the plugin will compress +http requests using gzip. + +[id="{version}-plugins-{type}s-{plugin}-http_method"] +===== `http_method` + + * This is a required setting. + * Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head` + * There is no default value for this setting. + +The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head" + +[id="{version}-plugins-{type}s-{plugin}-ignorable_codes"] +===== `ignorable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +If you would like to consider some non-2xx codes to be successes +enumerate them here. Responses returning these codes will be considered successes + +[id="{version}-plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="{version}-plugins-{type}s-{plugin}-mapping"] +===== `mapping` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * There is no default value for this setting. + +This lets you choose the structure and parts of the event that are sent. + + +For example: +[source,ruby] + mapping => {"foo" => "%{host}" + "bar" => "%{type}"} + +[id="{version}-plugins-{type}s-{plugin}-message"] +===== `message` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + + + +[id="{version}-plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="{version}-plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="{version}-plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60` + +This module makes it easy to add a very fully configured HTTP client to logstash +based on [Manticore](https://github.com/cheald/manticore). +For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller +Timeout (in seconds) for the entire request + +[id="{version}-plugins-{type}s-{plugin}-retry_failed"] +===== `retry_failed` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Note that this option controls plugin-level retries only. +It has no affect on library-level retries. + +Set this option to `false` if you want to disable infinite retries for HTTP error response codes defined in the <<{version}-plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> or +retryable exceptions (Timeout, SocketException, ClientProtocolException, ResolutionFailure and SocketTimeout). +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When this option is set to `false` and `automatic_retries` is enabled, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +When set to `true` and `automatic_retries` is enabled, this will cause non-idempotent HTTP verbs (such as POST) to be retried. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-retryable_codes"] +===== `retryable_codes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `[429, 500, 502, 503, 504]` + +If the plugin encounters these response codes, the plugin will retry indefinitely. +See <<{version}-plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information. + +[id="{version}-plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_key>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting + +The .cer or .pem CA files to validate the server's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable SSL/TLS secured communication. It must be `true` for other `ssl_` options +to take effect. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +OpenSSL-style RSA private key that corresponds to the <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> is set. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Supported values are: `full`, `none` + * Default value is `full` + +Controls the verification of server certificates. +The `full` option verifies that the provided certificate is signed by a trusted authority (CA) +and also that the server’s hostname (or IP address) matches the names identified within the certificate. + +The `none` setting performs no verification of the server’s certificate. +This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration. +It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors. +Using `none` in production environments is strongly discouraged. + +[id="{version}-plugins-{type}s-{plugin}-url"] +===== `url` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URL to use + +[id="{version}-plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `200` + +How long to wait before checking if the connection is stale before executing a request on a connection using keepalive. +You may want to set this lower, possibly to 0 if you get connection errors regularly +Quoting the Apache commons docs (this client is based Apache Commmons): +'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.' +See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info] + + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== HTTP Output Obsolete Configuration Options + +WARNING: As of version `6.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| cacert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| client_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| client_key |<<{version}-plugins-{type}s-{plugin}-ssl_key>> +| keystore |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +| keystore_password |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| keystore_type |<<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> +| truststore |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +| truststore_password |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> +| truststore_type |<<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/kafka-index.asciidoc b/docs/versioned-plugins/outputs/kafka-index.asciidoc index 4fe692463..8683aa0bf 100644 --- a/docs/versioned-plugins/outputs/kafka-index.asciidoc +++ b/docs/versioned-plugins/outputs/kafka-index.asciidoc @@ -5,6 +5,21 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-08-28 +| <> | 2025-06-12 +| <> | 2025-05-21 +| <> | 2025-05-08 +| <> | 2025-01-07 +| <> | 2024-12-18 +| <> | 2024-11-25 +| <> | 2024-10-04 +| <> | 2024-07-26 +| <> | 2024-07-02 +| <> | 2024-05-17 +| <> | 2024-04-10 +| <> | 2024-04-09 +| <> | 2024-02-06 +| <> | 2024-01-09 | <> | 2023-10-16 | <> | 2023-10-04 | <> | 2023-07-04 @@ -12,6 +27,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-02-14 | <> | 2023-01-25 | <> | 2022-12-16 +| <> | 2025-02-12 | <> | 2023-10-16 | <> | 2022-05-09 | <> | 2022-04-25 @@ -66,6 +82,21 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-11 |======================================================================= +include::kafka-v11.6.4.asciidoc[] +include::kafka-v11.6.3.asciidoc[] +include::kafka-v11.6.2.asciidoc[] +include::kafka-v11.6.1.asciidoc[] +include::kafka-v11.6.0.asciidoc[] +include::kafka-v11.5.4.asciidoc[] +include::kafka-v11.5.3.asciidoc[] +include::kafka-v11.5.2.asciidoc[] +include::kafka-v11.5.1.asciidoc[] +include::kafka-v11.5.0.asciidoc[] +include::kafka-v11.4.2.asciidoc[] +include::kafka-v11.4.1.asciidoc[] +include::kafka-v11.4.0.asciidoc[] +include::kafka-v11.3.4.asciidoc[] +include::kafka-v11.3.3.asciidoc[] include::kafka-v11.3.2.asciidoc[] include::kafka-v11.3.1.asciidoc[] include::kafka-v11.3.0.asciidoc[] @@ -73,6 +104,7 @@ include::kafka-v11.2.1.asciidoc[] include::kafka-v11.2.0.asciidoc[] include::kafka-v11.1.0.asciidoc[] include::kafka-v11.0.0.asciidoc[] +include::kafka-v10.12.2.asciidoc[] include::kafka-v10.12.1.asciidoc[] include::kafka-v10.12.0.asciidoc[] include::kafka-v10.11.0.asciidoc[] diff --git a/docs/versioned-plugins/outputs/kafka-v10.12.2.asciidoc b/docs/versioned-plugins/outputs/kafka-v10.12.2.asciidoc new file mode 100644 index 000000000..ac4ba0369 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v10.12.2.asciidoc @@ -0,0 +1,510 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 2.8 +:kafka_client_doc: 25 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v10.12.2 +:release_date: 2025-02-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v10.12.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +the use of this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.3.3.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.3.3.asciidoc new file mode 100644 index 000000000..bc175d577 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.3.3.asciidoc @@ -0,0 +1,516 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.3.3 +:release_date: 2024-01-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.3.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If explicitly configured it fallbacks to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +the use of this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.3.4.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.3.4.asciidoc new file mode 100644 index 000000000..9779d8ab8 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.3.4.asciidoc @@ -0,0 +1,516 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.3.4 +:release_date: 2024-02-06 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.3.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.4.0.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.4.0.asciidoc new file mode 100644 index 000000000..3b4864f34 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.4.0.asciidoc @@ -0,0 +1,533 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.0 +:release_date: 2024-04-09 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_header>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_header => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.4.1.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.4.1.asciidoc new file mode 100644 index 000000000..f760bca32 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.4.1.asciidoc @@ -0,0 +1,533 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.1 +:release_date: 2024-04-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.4.2.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.4.2.asciidoc new file mode 100644 index 000000000..7fe1343bf --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.4.2.asciidoc @@ -0,0 +1,533 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.4.2 +:release_date: 2024-05-17 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.4.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.5.0.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.5.0.asciidoc new file mode 100644 index 000000000..a04cef3c4 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.5.0.asciidoc @@ -0,0 +1,533 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.0 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.5.1.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.5.1.asciidoc new file mode 100644 index 000000000..da95c451f --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.5.1.asciidoc @@ -0,0 +1,533 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.1 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.5.2.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.5.2.asciidoc new file mode 100644 index 000000000..c6a6eeae8 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.5.2.asciidoc @@ -0,0 +1,541 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.4 +:kafka_client_doc: 34 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.2 +:release_date: 2024-10-04 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class""] +===== `sasl_client_callback_handler_class` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.5.3.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.5.3.asciidoc new file mode 100644 index 000000000..e659101df --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.5.3.asciidoc @@ -0,0 +1,541 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.7 +:kafka_client_doc: 37 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.3 +:release_date: 2024-11-25 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class""] +===== `sasl_client_callback_handler_class` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.5.4.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.5.4.asciidoc new file mode 100644 index 000000000..cc28b89b0 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.5.4.asciidoc @@ -0,0 +1,541 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.5.4 +:release_date: 2024-12-18 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.5.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class""] +===== `sasl_client_callback_handler_class` +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.6.0.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.6.0.asciidoc new file mode 100644 index 000000000..7af92c7d1 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.6.0.asciidoc @@ -0,0 +1,597 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.0 +:release_date: 2025-01-07 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.6.1.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.6.1.asciidoc new file mode 100644 index 000000000..4fbc23641 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.6.1.asciidoc @@ -0,0 +1,624 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.1 +:release_date: 2025-05-08 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is <> +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.6.2.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.6.2.asciidoc new file mode 100644 index 000000000..6014a4206 --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.6.2.asciidoc @@ -0,0 +1,624 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.8.1 +:kafka_client_doc: 38 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.2 +:release_date: 2025-05-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.6.3.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.6.3.asciidoc new file mode 100644 index 000000000..253bf478b --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.6.3.asciidoc @@ -0,0 +1,624 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.9.1 +:kafka_client_doc: 39 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.3 +:release_date: 2025-06-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/kafka-v11.6.4.asciidoc b/docs/versioned-plugins/outputs/kafka-v11.6.4.asciidoc new file mode 100644 index 000000000..506760ffb --- /dev/null +++ b/docs/versioned-plugins/outputs/kafka-v11.6.4.asciidoc @@ -0,0 +1,624 @@ +:integration: kafka +:plugin: kafka +:type: output +:default_codec: plain +:kafka_client: 3.9.1 +:kafka_client_doc: 39 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v11.6.4 +:release_date: 2025-08-28 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-kafka/blob/v11.6.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Kafka output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Write events to a Kafka topic. + +This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the +official +https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka +compatibility reference]. If the linked compatibility wiki is not up-to-date, +please contact Kafka support/community to confirm compatibility. + +If you require features not yet available in this plugin (including client +version upgrades), please file an issue with details about what you need. + +This output supports connecting to Kafka over: + +* SSL (requires plugin version 3.0.0 or later) +* Kerberos SASL (requires plugin version 5.1.0 or later) + +By default security is disabled but can be turned on as needed. + +The only required configuration is the topic_id. + +The default codec is plain. Logstash will encode your events with not only the +message field but also with a timestamp and hostname. + +If you want the full content of your events to be sent as json, you should set +the codec in the output configuration like this: + +[source,ruby] + output { + kafka { + codec => json + topic_id => "mytopic" + } + } + +For more information see +https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer + +Kafka producer configuration: +https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs + +NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. + +[id="{version}-plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Kafka Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting, +and might change if Kafka's producer defaults change. +See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-acks>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["0", "1", "all"]`|No +| <<{version}-plugins-{type}s-{plugin}-batch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bootstrap_servers>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-buffer_memory>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-client_dns_lookup>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-client_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-compression_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No +| <<{version}-plugins-{type}s-{plugin}-connections_max_idle_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-key_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-linger_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-max_request_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-message_headers>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-message_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-metadata_max_age_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-partitioner>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-receive_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-request_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retries>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_jaas_config>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sasl_mechanism>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-security_protocol>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No +| <<{version}-plugins-{type}s-{plugin}-send_buffer_bytes>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-topic_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-value_serializer>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-acks"] +===== `acks` + + * Value can be any of: `0`, `1`, `all` + * Default value is `"1"` + +The number of acknowledgments the producer requires the leader to have received +before considering a request complete. + +`acks=0`. The producer will not wait for any acknowledgment from the server. + +`acks=1`. The leader will write the record to its local log, but will respond +without waiting for full acknowledgement from all followers. + +`acks=all`. The leader will wait for the full set of in-sync replicas before +acknowledging the record. + +[id="{version}-plugins-{type}s-{plugin}-batch_size"] +===== `batch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `16384`. + +The producer will attempt to batch records together into fewer requests whenever multiple +records are being sent to the same partition. This helps performance on both the client +and the server. This configuration controls the default batch size in bytes. + +[id="{version}-plugins-{type}s-{plugin}-bootstrap_servers"] +===== `bootstrap_servers` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"localhost:9092"` + +This is for bootstrapping and the producer will only use it for getting metadata (topics, +partitions and replicas). The socket connections for sending the actual data will be +established based on the broker information returned in the metadata. The format is +`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a +subset of brokers. + +[id="{version}-plugins-{type}s-{plugin}-buffer_memory"] +===== `buffer_memory` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `33554432` (32MB). + +The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + +[id="{version}-plugins-{type}s-{plugin}-client_dns_lookup"] +===== `client_dns_lookup` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default` + * Default value is `"default"` + +Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries +all IP addresses returned for a hostname before failing the connection. +If set to `resolve_canonical_bootstrap_servers_only`, each entry will be +resolved and expanded into a list of canonical names. + +[NOTE] +==== +Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. +If not explicitly configured it defaults to `use_all_dns_ips`. +==== + +[id="{version}-plugins-{type}s-{plugin}-client_id"] +===== `client_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The id string to pass to the server when making requests. +The purpose of this is to be able to track the source of requests beyond just +ip/port by allowing a logical application name to be included with the request + +[id="{version}-plugins-{type}s-{plugin}-compression_type"] +===== `compression_type` + + * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd` + * Default value is `"none"` + +The compression type for all data generated by the producer. +The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd. + +[id="{version}-plugins-{type}s-{plugin}-connections_max_idle_ms"] +===== `connections_max_idle_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `540000` milliseconds (9 minutes). + +Close idle connections after the number of milliseconds specified by this config. + +[id="{version}-plugins-{type}s-{plugin}-jaas_path"] +===== `jaas_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization +services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: +[source,java] +---------------------------------- +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="kafka"; + }; +---------------------------------- + +Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these +to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same +`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on +different JVM instances. + +[id="{version}-plugins-{type}s-{plugin}-kerberos_config"] +===== `kerberos_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html + +[id="{version}-plugins-{type}s-{plugin}-key_serializer"] +===== `key_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the key of the message + +[id="{version}-plugins-{type}s-{plugin}-linger_ms"] +===== `linger_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The producer groups together any records that arrive in between request +transmissions into a single batched request. Normally this occurs only under +load when records arrive faster than they can be sent out. However in some circumstances +the client may want to reduce the number of requests even under moderate load. +This setting accomplishes this by adding a small amount of artificial delay—that is, +rather than immediately sending out a record the producer will wait for up to the given delay +to allow other records to be sent so that the sends can be batched together. + +[id="{version}-plugins-{type}s-{plugin}-max_request_size"] +===== `max_request_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1048576` (1MB). + +The maximum size of a request + +[id="{version}-plugins-{type}s-{plugin}-message_headers"] +===== `message_headers` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + ** Keys are header names, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values are header values, and must be {logstash-ref}/configuration-file-structure.html#string[string] + ** Values support interpolation from event field values + * There is no default value for this setting. + +A map of key value pairs, each corresponding to a header name and its value respectively. +Example: +[source,ruby] +---------------------------------- + message_headers => { "event_timestamp" => "%{@timestamp}" } +---------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-message_key"] +===== `message_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key for the message. + +[id="{version}-plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"] +===== `metadata_fetch_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `60000` milliseconds (60 seconds). + +The timeout setting for initial metadata request to fetch topic metadata. + +[id="{version}-plugins-{type}s-{plugin}-metadata_max_age_ms"] +===== `metadata_max_age_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `300000` milliseconds (5 minutes). + +The max time in milliseconds before a metadata refresh is forced. + +[id="{version}-plugins-{type}s-{plugin}-partitioner"] +===== `partitioner` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. + +The default behavior is to hash the `message_key` of an event to get the partition. +When no message key is present, the plugin picks a partition in a round-robin fashion. + +Available options for choosing a partitioning strategy are as follows: + +* `default` use the default partitioner as described above +* `round_robin` distributes writes to all partitions equally, regardless of `message_key` +* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one + +[id="{version}-plugins-{type}s-{plugin}-receive_buffer_bytes"] +===== `receive_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `32768` (32KB). + +The size of the TCP receive buffer to use when reading data + +[id="{version}-plugins-{type}s-{plugin}-reconnect_backoff_ms"] +===== `reconnect_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50`. + +The amount of time to wait before attempting to reconnect to a given host when a connection fails. + +[id="{version}-plugins-{type}s-{plugin}-request_timeout_ms"] +===== `request_timeout_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `40000` milliseconds (40 seconds). + +The configuration controls the maximum amount of time the client will wait +for the response of a request. If the response is not received before the timeout +elapses the client will resend the request if necessary or fail the request if +retries are exhausted. + +[id="{version}-plugins-{type}s-{plugin}-retries"] +===== `retries` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default retry behavior is to retry until successful. To prevent data loss, +changing this setting is discouraged. + +If you choose to set `retries`, a value greater than zero will cause the +client to only retry a fixed number of times. This will result in data loss +if a transport fault exists for longer than your retry count (network outage, +Kafka down, etc). + +A value less than zero is a configuration error. + +Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException] +and +https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException]. +If producing a message throws any other exception, an error is logged and the message is dropped without retrying. +This prevents the Logstash pipeline from hanging indefinitely. + +In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured. + +[id="{version}-plugins-{type}s-{plugin}-retry_backoff_ms"] +===== `retry_backoff_ms` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +The amount of time to wait before attempting to retry a failed produce request to a given topic partition. + +[id="{version}-plugins-{type}s-{plugin}-sasl_client_callback_handler_class"] +===== `sasl_client_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL client callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"] +===== `sasl_oauthbearer_token_endpoint_url` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The URL for the OAuth 2.0 issuer token endpoint. + +[id="{version}-plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"] +===== `sasl_oauthbearer_scope_claim_name` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"scope"` + +(optional) The override name of the scope claim. + +[id="{version}-plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is {logstash-ref}/configuration-file-structure.html#array[array] +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] +===== `sasl_login_callback_handler_class` + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The SASL login callback handler class the specified SASL mechanism should use. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"] +===== `sasl_login_connect_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS connect timeout + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"] +===== `sasl_login_read_timeout_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +(optional) The duration, in milliseconds, for HTTPS read timeout. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"] +===== `sasl_login_retry_backoff_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100` milliseconds. + +(optional) The duration, in milliseconds, to wait between HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"] +===== `sasl_login_retry_backoff_max_ms` + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10000` milliseconds. + +(optional) The maximum duration, in milliseconds, for HTTPS call attempts. + +[id="{version}-plugins-{type}s-{plugin}-sasl_jaas_config"] +===== `sasl_jaas_config` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. + +If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence. + +Example (setting for Azure Event Hub): +[source,ruby] + output { + kafka { + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser' password='apassword';" + } + } + +[id="{version}-plugins-{type}s-{plugin}-sasl_kerberos_service_name"] +===== `sasl_kerberos_service_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The Kerberos principal name that Kafka broker runs as. +This can be defined either in Kafka's JAAS config or in Kafka's config. + +[id="{version}-plugins-{type}s-{plugin}-sasl_mechanism"] +===== `sasl_mechanism` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"GSSAPI"` + +http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. +This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. +GSSAPI is the default mechanism. + +[id="{version}-plugins-{type}s-{plugin}-security_protocol"] +===== `security_protocol` + + * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` + * Default value is `"PLAINTEXT"` + +Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL + +[id="{version}-plugins-{type}s-{plugin}-send_buffer_bytes"] +===== `send_buffer_bytes` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `131072` (128KB). + +The size of the TCP send buffer to use when sending data. + +[id="{version}-plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"] +===== `ssl_endpoint_identification_algorithm` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"https"` + +The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_password"] +===== `ssl_key_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The password of the private key in the key store file. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_location"] +===== `ssl_keystore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore path. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +If client authentication is required, this setting stores the keystore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The keystore type. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_location"] +===== `ssl_truststore_location` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The JKS truststore path to validate the Kafka broker's certificate. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +The truststore password + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The truststore type. + +[id="{version}-plugins-{type}s-{plugin}-topic_id"] +===== `topic_id` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The topic to produce messages to + +[id="{version}-plugins-{type}s-{plugin}-value_serializer"] +===== `value_serializer` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"org.apache.kafka.common.serialization.StringSerializer"` + +Serializer class for the value of the message + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/logstash-index.asciidoc b/docs/versioned-plugins/outputs/logstash-index.asciidoc index 34e33d3c0..790227396 100644 --- a/docs/versioned-plugins/outputs/logstash-index.asciidoc +++ b/docs/versioned-plugins/outputs/logstash-index.asciidoc @@ -5,6 +5,9 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-12-10 +| <> | 2024-09-12 +| <> | 2024-01-23 | <> | 2023-11-16 | <> | 2023-11-06 | <> | 2023-10-03 @@ -14,6 +17,9 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-09-25 |======================================================================= +include::logstash-v1.0.4.asciidoc[] +include::logstash-v1.0.3.asciidoc[] +include::logstash-v1.0.2.asciidoc[] include::logstash-v1.0.1.asciidoc[] include::logstash-v1.0.0.asciidoc[] include::logstash-v0.0.5.asciidoc[] diff --git a/docs/versioned-plugins/outputs/logstash-v1.0.2.asciidoc b/docs/versioned-plugins/outputs/logstash-v1.0.2.asciidoc new file mode 100644 index 000000000..e8609ace4 --- /dev/null +++ b/docs/versioned-plugins/outputs/logstash-v1.0.2.asciidoc @@ -0,0 +1,284 @@ +:integration: logstash +:plugin: logstash +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.2 +:release_date: 2024-01-23 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Send events to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] in a pipeline that may be in another process or on another host. +You must have a TCP route to the port (defaults to 9800) on an interface that the downstream input is bound to. + +NOTE: Sending events to _any_ destination other than a `logstash-input` plugin is neither advised nor supported. + We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding. + +[id="{version}-plugins-{type}s-{plugin}-minimum-config"] +===== Minimum Configuration +[cols="2a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +output { + logstash { + hosts => "10.0.0.123:9801" + } +} +---- + +| + +[source,shell] +---- +output { + logstash { + hosts => "10.0.0.123:9801" + ssl_enabled + => false + } +} +---- + +|======================================================================================================================= + +[id="{version}-plugins-{type}s-{plugin}-config-connecting"] +===== Configuration Concepts + +Configure this output plugin to connect to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] by specifying its `hosts`. +Depending on the downstream plugin's configuration, you may need to also configure the target port, SSL, and/or credentials. + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-trust"] +===== Security: SSL Trust + +When communicating over SSL, this plugin establishes trust of the server it connects to before transmitting credentials or events. + +It does so by ensuring that the responding server presents a currently-valid certificate with identity claims matching host it is connecting to, signed by a trusted signing authority, along with proof-of-possession of the associated private key material. + +The system trust store is used by default. +You can provide an _alternate_ source of trust with _ONE OF_: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A PKCS12- or JKS-formatted truststore (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-identity"] +===== Security: SSL Identity + +If the downstream input plugin is configured to request or require client authentication, you can configure this plugin to provide its proof-of-identity with _ONE OF_: + +* JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) +* PKCS8-formatted Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) + +[id="{version}-plugins-{type}s-{plugin}-config-credentials"] +===== Security: Credentials + +If the downstream `logstash-input` plugin is configured to require `username` and `password`, +you will need to configure this output with a matching <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>. + +NOTE: when SSL is disabled, data and credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Logstash Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type |Required +| <<{version}-plugins-{type}s-{plugin}-hosts>> |list of {logstash-ref}/configuration-file-structure.html#string[string] |Yes +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is list of {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* Constraints: +** When using IPv6, IP address must be in an enclosed in brackets. +** When a port is not provided, the default `9800` is used. + +The addresses of one or more downstream `input`s to connect to. + +Host can be any of IPv4, IPv6 (in enclosed bracket) or host name, examples: + +* `"127.0.0.1"` +* `"127.0.0.1:9801"` +* `"ds.example.com"` +* `"ds.example.com:9802"` +* `"[::1]"` +* `"[::1]:9803"` +* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"` +* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9804"` + +Plugin balances incoming load among the `hosts`. For more information, visit {logstash-ref}/plugins-integrations-logstash.html[Logstash integration plugin] _Load Balancing_ section. + +When connecting, communication to downstream input {ls} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. +============ + +NOTE: when using SSL, the server that responds must present a certificated with identity claim matching this host name or ip address. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when <<{version}-plugins-{type}s-{plugin}-username>> is configured. + +Password for password-based authentication. + +When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Logstash-to-Logstash communication is secured by default. +When the downstream input plugin disables SSL, it must also be disabled here. + +You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> is also required. +* Cannot be combined with configurations that disable SSL. + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting downstream input. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>. + +One or more PEM-encoded files defining certificate authorities for use in downstream input authentication. +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by downstream input. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* Cannot be combined with configurations that disable SSL. + +A path to an PEM-encoded _unencrypted_ PKCS8 SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* Cannot be combined with configurations that disable SSL. + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to the downstream input. +The provided identity will be used if the downstream input enables <<{version}-plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> is also required. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>. + +A path to a JKS- or PKCS12-formatted truststore with which to validate the identity claims of the downstream input. +The provided identity will be used if the downstream input enables <<{version}-plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* The supported modes are: +** `full`: verifies that a certificate provided by the client has an identity claim matching <<{version}-plugins-{type}s-{plugin}-hosts>>, is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key. +** `none`: performs no validation of the presented certificate + +* The default value is `full`. +* Cannot be combined with configurations that disable SSL. + +When communicating over SSL, this setting controls how the downstream input's certificate is verified. + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +Username for password-based authentication. + +When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair. + +NOTE: when SSL is disabled, credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/logstash-v1.0.3.asciidoc b/docs/versioned-plugins/outputs/logstash-v1.0.3.asciidoc new file mode 100644 index 000000000..917a5683b --- /dev/null +++ b/docs/versioned-plugins/outputs/logstash-v1.0.3.asciidoc @@ -0,0 +1,284 @@ +:integration: logstash +:plugin: logstash +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.3 +:release_date: 2024-09-12 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.3/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Send events to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] in a pipeline that may be in another process or on another host. +You must have a TCP route to the port (defaults to 9800) on an interface that the downstream input is bound to. + +NOTE: Sending events to _any_ destination other than a `logstash-input` plugin is neither advised nor supported. + We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding. + +[id="{version}-plugins-{type}s-{plugin}-minimum-config"] +===== Minimum Configuration +[cols="2a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +output { + logstash { + hosts => "10.0.0.123:9801" + } +} +---- + +| + +[source,shell] +---- +output { + logstash { + hosts => "10.0.0.123:9801" + ssl_enabled + => false + } +} +---- + +|======================================================================================================================= + +[id="{version}-plugins-{type}s-{plugin}-config-connecting"] +===== Configuration Concepts + +Configure this output plugin to connect to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] by specifying its `hosts`. +Depending on the downstream plugin's configuration, you may need to also configure the target port, SSL, and/or credentials. + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-trust"] +===== Security: SSL Trust + +When communicating over SSL, this plugin establishes trust of the server it connects to before transmitting credentials or events. + +It does so by ensuring that the responding server presents a currently-valid certificate with identity claims matching host it is connecting to, signed by a trusted signing authority, along with proof-of-possession of the associated private key material. + +The system trust store is used by default. +You can provide an _alternate_ source of trust with _ONE OF_: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A PKCS12- or JKS-formatted truststore (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-identity"] +===== Security: SSL Identity + +If the downstream input plugin is configured to request or require client authentication, you can configure this plugin to provide its proof-of-identity with _ONE OF_: + +* JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) +* PKCS8-formatted Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) + +[id="{version}-plugins-{type}s-{plugin}-config-credentials"] +===== Security: Credentials + +If the downstream `logstash-input` plugin is configured to require `username` and `password`, +you will need to configure this output with a matching <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>. + +NOTE: when SSL is disabled, data and credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Logstash Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type |Required +| <<{version}-plugins-{type}s-{plugin}-hosts>> |list of {logstash-ref}/configuration-file-structure.html#string[string] |Yes +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is list of {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* Constraints: +** When using IPv6, IP address must be in an enclosed in brackets. +** When a port is not provided, the default `9800` is used. + +The addresses of one or more downstream `input`s to connect to. + +Host can be any of IPv4, IPv6 (in enclosed bracket) or host name, examples: + +* `"127.0.0.1"` +* `"127.0.0.1:9801"` +* `"ds.example.com"` +* `"ds.example.com:9802"` +* `"[::1]"` +* `"[::1]:9803"` +* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"` +* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9804"` + +Plugin balances incoming load among the `hosts`. For more information, visit {logstash-ref}/plugins-integrations-logstash.html[Logstash integration plugin] _Load Balancing_ section. + +When connecting, communication to downstream input {ls} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. +============ + +NOTE: when using SSL, the server that responds must present a certificated with identity claim matching this host name or ip address. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when <<{version}-plugins-{type}s-{plugin}-username>> is configured. + +Password for password-based authentication. + +When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Logstash-to-Logstash communication is secured by default. +When the downstream input plugin disables SSL, it must also be disabled here. + +You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> is also required. +* Cannot be combined with configurations that disable SSL. + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting downstream input. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>. + +One or more PEM-encoded files defining certificate authorities for use in downstream input authentication. +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by downstream input. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* Cannot be combined with configurations that disable SSL. + +A path to an PEM-encoded _unencrypted_ PKCS8 SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* Cannot be combined with configurations that disable SSL. + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to the downstream input. +The provided identity will be used if the downstream input enables <<{version}-plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> is also required. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>. + +A path to a JKS- or PKCS12-formatted truststore with which to validate the identity claims of the downstream input. +The provided identity will be used if the downstream input enables <<{version}-plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* The supported modes are: +** `full`: verifies that a certificate provided by the client has an identity claim matching <<{version}-plugins-{type}s-{plugin}-hosts>>, is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key. +** `none`: performs no validation of the presented certificate + +* The default value is `full`. +* Cannot be combined with configurations that disable SSL. + +When communicating over SSL, this setting controls how the downstream input's certificate is verified. + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +Username for password-based authentication. + +When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair. + +NOTE: when SSL is disabled, credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/logstash-v1.0.4.asciidoc b/docs/versioned-plugins/outputs/logstash-v1.0.4.asciidoc new file mode 100644 index 000000000..a0dd949eb --- /dev/null +++ b/docs/versioned-plugins/outputs/logstash-v1.0.4.asciidoc @@ -0,0 +1,284 @@ +:integration: logstash +:plugin: logstash +:type: output +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v1.0.4 +:release_date: 2024-12-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-logstash/blob/v1.0.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Logstash output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Send events to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] in a pipeline that may be in another process or on another host. +You must have a TCP route to the port (defaults to 9800) on an interface that the downstream input is bound to. + +NOTE: Sending events to _any_ destination other than a `logstash-input` plugin is neither advised nor supported. + We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding. + +[id="{version}-plugins-{type}s-{plugin}-minimum-config"] +===== Minimum Configuration +[cols="2a,2a"] +|======================================================================================================================= +|SSL Enabled |SSL Disabled + +| + +[source,shell] +---- +output { + logstash { + hosts => "10.0.0.123:9801" + } +} +---- + +| + +[source,shell] +---- +output { + logstash { + hosts => "10.0.0.123:9801" + ssl_enabled + => false + } +} +---- + +|======================================================================================================================= + +[id="{version}-plugins-{type}s-{plugin}-config-connecting"] +===== Configuration Concepts + +Configure this output plugin to connect to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] by specifying its `hosts`. +Depending on the downstream plugin's configuration, you may need to also configure the target port, SSL, and/or credentials. + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-trust"] +===== Security: SSL Trust + +When communicating over SSL, this plugin establishes trust of the server it connects to before transmitting credentials or events. + +It does so by ensuring that the responding server presents a currently-valid certificate with identity claims matching host it is connecting to, signed by a trusted signing authority, along with proof-of-possession of the associated private key material. + +The system trust store is used by default. +You can provide an _alternate_ source of trust with _ONE OF_: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A PKCS12- or JKS-formatted truststore (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +[id="{version}-plugins-{type}s-{plugin}-config-ssl-identity"] +===== Security: SSL Identity + +If the downstream input plugin is configured to request or require client authentication, you can configure this plugin to provide its proof-of-identity with _ONE OF_: + +* JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) +* PKCS8-formatted Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) + +[id="{version}-plugins-{type}s-{plugin}-config-credentials"] +===== Security: Credentials + +If the downstream `logstash-input` plugin is configured to require `username` and `password`, +you will need to configure this output with a matching <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>. + +NOTE: when SSL is disabled, data and credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Logstash Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type |Required +| <<{version}-plugins-{type}s-{plugin}-hosts>> |list of {logstash-ref}/configuration-file-structure.html#string[string] |Yes +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is list of {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* Constraints: +** When using IPv6, IP address must be in an enclosed in brackets. +** When a port is not provided, the default `9800` is used. + +The addresses of one or more downstream `input`s to connect to. + +Host can be any of IPv4, IPv6 (in enclosed bracket) or host name, examples: + +* `"127.0.0.1"` +* `"127.0.0.1:9801"` +* `"ds.example.com"` +* `"ds.example.com:9802"` +* `"[::1]"` +* `"[::1]:9803"` +* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"` +* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9804"` + +Plugin balances incoming load among the `hosts`. For more information, visit {logstash-ref}/plugins-integrations-logstash.html[Logstash integration plugin] _Load Balancing_ section. + +When connecting, communication to downstream input {ls} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. +============ + +NOTE: when using SSL, the server that responds must present a certificated with identity claim matching this host name or ip address. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when <<{version}-plugins-{type}s-{plugin}-username>> is configured. + +Password for password-based authentication. + +When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `true` + +Logstash-to-Logstash communication is secured by default. +When the downstream input plugin disables SSL, it must also be disabled here. + +You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> is also required. +* Cannot be combined with configurations that disable SSL. + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting downstream input. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>. + +One or more PEM-encoded files defining certificate authorities for use in downstream input authentication. +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by downstream input. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* Cannot be combined with configurations that disable SSL. + +A path to an PEM-encoded _unencrypted_ PKCS8 SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* Cannot be combined with configurations that disable SSL. + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to the downstream input. +The provided identity will be used if the downstream input enables <<{version}-plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> is also required. +* Cannot be combined with configurations that disable SSL. +* Cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>. + +A path to a JKS- or PKCS12-formatted truststore with which to validate the identity claims of the downstream input. +The provided identity will be used if the downstream input enables <<{version}-plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* Cannot be combined with configurations that disable SSL. + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* The supported modes are: +** `full`: verifies that a certificate provided by the client has an identity claim matching <<{version}-plugins-{type}s-{plugin}-hosts>>, is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key. +** `none`: performs no validation of the presented certificate + +* The default value is `full`. +* Cannot be combined with configurations that disable SSL. + +When communicating over SSL, this setting controls how the downstream input's certificate is verified. + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +Username for password-based authentication. + +When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair. + +NOTE: when SSL is disabled, credentials will be transmitted in clear-text. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:no_codec!: diff --git a/docs/versioned-plugins/outputs/mongodb-index.asciidoc b/docs/versioned-plugins/outputs/mongodb-index.asciidoc index 234417014..809520978 100644 --- a/docs/versioned-plugins/outputs/mongodb-index.asciidoc +++ b/docs/versioned-plugins/outputs/mongodb-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-01-02 | <> | 2022-01-31 | <> | 2019-04-29 | <> | 2019-03-25 @@ -14,6 +15,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::mongodb-v3.1.8.asciidoc[] include::mongodb-v3.1.7.asciidoc[] include::mongodb-v3.1.6.asciidoc[] include::mongodb-v3.1.5.asciidoc[] diff --git a/docs/versioned-plugins/outputs/mongodb-v3.1.8.asciidoc b/docs/versioned-plugins/outputs/mongodb-v3.1.8.asciidoc new file mode 100644 index 000000000..2002f5a38 --- /dev/null +++ b/docs/versioned-plugins/outputs/mongodb-v3.1.8.asciidoc @@ -0,0 +1,137 @@ +:plugin: mongodb +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.1.8 +:release_date: 2025-01-02 +:changelog_url: https://github.com/logstash-plugins/logstash-output-mongodb/blob/v3.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Mongodb output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output writes events to MongoDB. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Mongodb Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-bulk>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-bulk_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-collection>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-database>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-generateId>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-isodate>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-bulk"] +===== `bulk` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Bulk insert flag, set to true to allow bulk insertion, else it will insert events one by one. + +[id="{version}-plugins-{type}s-{plugin}-bulk_interval"] +===== `bulk_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `2` + +Bulk interval, Used to insert events periodically if the "bulk" flag is activated. + +[id="{version}-plugins-{type}s-{plugin}-bulk_size"] +===== `bulk_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `900` + +Bulk events number, if the number of events to insert into a collection raise that limit, it will be bulk inserted +whatever the bulk interval value (mongodb hard limit is 1000). + +[id="{version}-plugins-{type}s-{plugin}-collection"] +===== `collection` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The collection to use. This value can use `%{foo}` values to dynamically +select a collection based on data in the event. + +[id="{version}-plugins-{type}s-{plugin}-database"] +===== `database` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The database to use. + +[id="{version}-plugins-{type}s-{plugin}-generateId"] +===== `generateId` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If true, an "_id" field will be added to the document before insertion. +The "_id" field will use the timestamp of the event and overwrite an existing +"_id" field in the event. + +[id="{version}-plugins-{type}s-{plugin}-isodate"] +===== `isodate` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +If true, store the @timestamp field in MongoDB as an ISODate type instead +of an ISO8601 string. For more information about this, see +http://www.mongodb.org/display/DOCS/Dates. + +[id="{version}-plugins-{type}s-{plugin}-retry_delay"] +===== `retry_delay` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3` + +The number of seconds to wait after failure before retrying. + +[id="{version}-plugins-{type}s-{plugin}-uri"] +===== `uri` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +A MongoDB URI to connect to. +See http://docs.mongodb.org/manual/reference/connection-string/. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/rabbitmq-index.asciidoc b/docs/versioned-plugins/outputs/rabbitmq-index.asciidoc index b8632c868..9ad30c523 100644 --- a/docs/versioned-plugins/outputs/rabbitmq-index.asciidoc +++ b/docs/versioned-plugins/outputs/rabbitmq-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-09-16 | <> | 2023-06-09 | <> | 2023-05-30 | <> | 2022-10-04 @@ -28,6 +29,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::rabbitmq-v7.4.0.asciidoc[] include::rabbitmq-v7.3.3.asciidoc[] include::rabbitmq-v7.3.2.asciidoc[] include::rabbitmq-v7.3.1.asciidoc[] diff --git a/docs/versioned-plugins/outputs/rabbitmq-v7.4.0.asciidoc b/docs/versioned-plugins/outputs/rabbitmq-v7.4.0.asciidoc new file mode 100644 index 000000000..46fd30c9e --- /dev/null +++ b/docs/versioned-plugins/outputs/rabbitmq-v7.4.0.asciidoc @@ -0,0 +1,274 @@ +:integration: rabbitmq +:plugin: rabbitmq +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.4.0 +:release_date: 2024-09-16 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-rabbitmq/blob/v7.4.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Rabbitmq output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x +or later version (3.x is recommended). + +Relevant links: + +* http://www.rabbitmq.com/[RabbitMQ] +* http://rubymarchhare.info[March Hare] + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Rabbitmq Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-arguments>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-automatic_recovery>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-connect_retry_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-durable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-exchange>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-exchange_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fanout", "direct", "topic", "x-consistent-hash", "x-modulus-hash"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-heartbeat>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-message_properties>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-passive>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-persistent>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_password>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_path>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_version>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-vhost>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-arguments"] +===== `arguments` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +Extra queue arguments as an array. +To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}` + +[id="{version}-plugins-{type}s-{plugin}-automatic_recovery"] +===== `automatic_recovery` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Set this to automatically recover from a broken connection. You almost certainly don't want to override this!!! + +[id="{version}-plugins-{type}s-{plugin}-connect_retry_interval"] +===== `connect_retry_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Time in seconds to wait before retrying a connection + +[id="{version}-plugins-{type}s-{plugin}-connection_timeout"] +===== `connection_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +The default connection timeout in milliseconds. If not specified the timeout is infinite. + +[id="{version}-plugins-{type}s-{plugin}-durable"] +===== `durable` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Is this exchange durable? (aka; Should it survive a broker restart?) + +[id="{version}-plugins-{type}s-{plugin}-exchange"] +===== `exchange` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the exchange + +[id="{version}-plugins-{type}s-{plugin}-exchange_type"] +===== `exchange_type` + + * This is a required setting. + * Value can be any of: `fanout`, `direct`, `topic`, `x-consistent-hash`, `x-modulus-hash` + * There is no default value for this setting. + +The exchange type (fanout, topic, direct) + +[id="{version}-plugins-{type}s-{plugin}-heartbeat"] +===== `heartbeat` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +Heartbeat delay in seconds. If unspecified no heartbeats will be sent + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Common functionality for the rabbitmq input/output +RabbitMQ server address(es) +host can either be a single host, or a list of hosts +i.e. + host => "localhost" +or + host => ["host01", "host02] + +if multiple hosts are provided on the initial connection and any subsequent +recovery attempts of the hosts is chosen at random and connected to. +Note that only one host connection is active at a time. + +[id="{version}-plugins-{type}s-{plugin}-key"] +===== `key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +The default codec for this plugin is JSON. You can override this to suit your particular needs however. +Key to route to by default. Defaults to 'logstash' + +* Routing keys are ignored on fanout exchanges. + +[id="{version}-plugins-{type}s-{plugin}-message_properties"] +===== `message_properties` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Add properties to be set per-message here, such as 'content_type', 'priority'. +Values can be {logstash-ref}/event-dependent-configuration.html#sprintf[`sprintf` templates], whose value for each message will be populated from the event. + +Example: +[source,ruby] + message_properties => { + "content_type" => "application/json" + "priority" => 1 + } + + +[id="{version}-plugins-{type}s-{plugin}-passive"] +===== `passive` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Passive queue creation? Useful for checking queue existance without modifying server state + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `"guest"` + +RabbitMQ password + +[id="{version}-plugins-{type}s-{plugin}-persistent"] +===== `persistent` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Should RabbitMQ persist messages to disk? + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5672` + +RabbitMQ port to connect on + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * There is no default value for this setting. + +Enable or disable SSL. +Note that by default remote certificate verification is off. +Specify ssl_certificate_path and ssl_certificate_password if you need +certificate verification + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_password"] +===== `ssl_certificate_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Password for the encrypted PKCS12 (.p12) certificate file specified in ssl_certificate_path + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_path"] +===== `ssl_certificate_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to an SSL certificate in PKCS12 (.p12) format used for verifying the remote host + +[id="{version}-plugins-{type}s-{plugin}-ssl_version"] +===== `ssl_version` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"TLSv1.2"` + +Version of the SSL protocol to use. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"guest"` + +RabbitMQ username + +[id="{version}-plugins-{type}s-{plugin}-vhost"] +===== `vhost` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/"` + +The vhost (virtual host) to use. If you don't know what this +is, leave the default. With the exception of the default +vhost ("/"), names of vhosts should not begin with a forward +slash. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/redis-index.asciidoc b/docs/versioned-plugins/outputs/redis-index.asciidoc index 395ba4137..7bff20a98 100644 --- a/docs/versioned-plugins/outputs/redis-index.asciidoc +++ b/docs/versioned-plugins/outputs/redis-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-06-04 | <> | 2019-01-11 | <> | 2018-04-06 | <> | 2017-11-13 @@ -15,6 +16,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::redis-v5.2.0.asciidoc[] include::redis-v5.0.0.asciidoc[] include::redis-v4.0.4.asciidoc[] include::redis-v4.0.3.asciidoc[] diff --git a/docs/versioned-plugins/outputs/redis-v5.2.0.asciidoc b/docs/versioned-plugins/outputs/redis-v5.2.0.asciidoc new file mode 100644 index 000000000..acbc11c56 --- /dev/null +++ b/docs/versioned-plugins/outputs/redis-v5.2.0.asciidoc @@ -0,0 +1,300 @@ +:plugin: redis +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.2.0 +:release_date: 2024-06-04 +:changelog_url: https://github.com/logstash-plugins/logstash-output-redis/blob/v5.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Redis output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output will send events to a Redis queue using RPUSH. +The RPUSH command is supported in Redis v0.0.7+. Using +PUBLISH to a channel requires at least v1.3.8+. +While you may be able to make these Redis versions work, +the best performance and stability will be found in more +recent stable versions. Versions 2.6.0+ are recommended. + +For more information, see http://redis.io/[the Redis homepage] + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Redis Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-batch>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-batch_events>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-batch_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-congestion_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-congestion_threshold>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-data_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["list", "channel"]`|Yes +| <<{version}-plugins-{type}s-{plugin}-db>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-key>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-reconnect_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-shuffle_hosts>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |{logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-batch"] +===== `batch` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Set to true if you want Redis to batch up values and send 1 RPUSH command +instead of one command per value to push on the list. Note that this only +works with `data_type="list"` mode right now. + +If true, we send an RPUSH every "batch_events" events or +"batch_timeout" seconds (whichever comes first). +Only supported for `data_type` is "list". + +[id="{version}-plugins-{type}s-{plugin}-batch_events"] +===== `batch_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `50` + +If batch is set to true, the number of events we queue up for an RPUSH. + +[id="{version}-plugins-{type}s-{plugin}-batch_timeout"] +===== `batch_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +If batch is set to true, the maximum amount of time between RPUSH commands +when there are pending events to flush. + +[id="{version}-plugins-{type}s-{plugin}-congestion_interval"] +===== `congestion_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +How often to check for congestion. Default is one second. +Zero means to check on every event. + +[id="{version}-plugins-{type}s-{plugin}-congestion_threshold"] +===== `congestion_threshold` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +In case Redis `data_type` is `list` and has more than `@congestion_threshold` items, +block until someone consumes them and reduces congestion, otherwise if there are +no consumers Redis will run out of memory, unless it was configured with OOM protection. +But even with OOM protection, a single Redis list can block all other users of Redis, +until Redis CPU consumption reaches the max allowed RAM size. +A default value of 0 means that this limit is disabled. +Only supported for `list` Redis `data_type`. + +[id="{version}-plugins-{type}s-{plugin}-data_type"] +===== `data_type` + + * Value can be any of: `list`, `channel` + * There is no default value for this setting. + +Either list or channel. If `data_type` is list, then we will set +RPUSH to key. If `data_type` is channel, then we will PUBLISH to `key`. + +[id="{version}-plugins-{type}s-{plugin}-db"] +===== `db` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0` + +The Redis database number. + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["127.0.0.1"]` + +The hostname(s) of your Redis server(s). Ports may be specified on any +hostname, which will override the global port config. +If the hosts list is an array, Logstash will pick one random host to connect to, +if that host is disconnected it will then pick another. + +For example: +[source,ruby] + "127.0.0.1" + ["127.0.0.1", "127.0.0.2"] + ["127.0.0.1:6380", "127.0.0.1"] + +[id="{version}-plugins-{type}s-{plugin}-key"] +===== `key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of a Redis list or channel. Dynamic names are +valid here, for example `logstash-%{type}`. + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Password to authenticate with. There is no authentication by default. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `6379` + +The default port to connect on. Can be overridden on any hostname. + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL support. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Interval for reconnecting to failed Redis connections + +[id="{version}-plugins-{type}s-{plugin}-shuffle_hosts"] +===== `shuffle_hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Shuffle the host list during Logstash startup. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate the certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key path + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +[id="{version}-plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Redis initial connection timeout in seconds. + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/s3-index.asciidoc b/docs/versioned-plugins/outputs/s3-index.asciidoc index e966cef17..78ee187e8 100644 --- a/docs/versioned-plugins/outputs/s3-index.asciidoc +++ b/docs/versioned-plugins/outputs/s3-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -44,6 +48,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::s3-v7.2.1.asciidoc[] +include::s3-v7.2.0.asciidoc[] +include::s3-v7.1.8.asciidoc[] +include::s3-v7.1.7.asciidoc[] include::s3-v7.1.6.asciidoc[] include::s3-v7.1.5.asciidoc[] include::s3-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/outputs/s3-v7.1.7.asciidoc b/docs/versioned-plugins/outputs/s3-v7.1.7.asciidoc new file mode 100644 index 000000000..1ce3e548a --- /dev/null +++ b/docs/versioned-plugins/outputs/s3-v7.1.7.asciidoc @@ -0,0 +1,453 @@ +:integration: aws +:plugin: s3 +:type: output +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin batches and uploads logstash events into Amazon Simple Storage +Service (Amazon S3). + +IMPORTANT: The S3 output plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +S3 outputs create temporary files into the OS' temporary directory. +You can specify where to save them using the `temporary_directory` option. + +IMPORTANT: For configurations containing multiple s3 outputs with the restore +option enabled, each output should define its own 'temporary_directory'. + +===== Requirements + +* Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key) +* S3 PutObject permission + +===== S3 output file + +[source,txt] +----- +`ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt` +----- + +|======= +| ls.s3 | indicates logstash plugin s3 | +| 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. | +| 2013-04-18T10.00 | represents the time whenever you specify time_file. | +| tag_hello | indicates the event's tag. | +| part0 | If you indicate size_file, it will generate more parts if your file.size > size_file. +When a file is full, it gets pushed to the bucket and then deleted from the temporary directory. +If a file is empty, it is simply deleted. Empty files will not be pushed. | +|======= + +===== Crash Recovery + +This plugin will recover and upload temporary log files after crash/abnormal termination when using `restore` set to true + +===== Usage +This is an example of logstash config: +[source,ruby] +output { + s3{ + access_key_id => "crazy_key" (optional) + secret_access_key => "monkey_access_key" (optional) + region => "eu-west-1" (optional, default = "us-east-1") + bucket => "your_bucket" (required) + size_file => 2048 (optional) - Bytes + time_file => 5 (optional) - Minutes + codec => "plain" (optional) + canned_acl => "private" (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" ) + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== S3 Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-canned_acl>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"]`|No +| <<{version}-plugins-{type}s-{plugin}-encoding>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip"]`|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-restore>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-rotation_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["size_and_time", "size", "time"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["AES256", "aws:kms"]`|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-signature_version>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["v2", "v4"]`|No +| <<{version}-plugins-{type}s-{plugin}-size_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssekms_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_class>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"]`|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-time_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_multipart_threshold>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_workers_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +. External credentials file specified by `aws_credentials_file` +. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to S3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + output { + s3 { + access_key_id => "1234", + secret_access_key => "secret", + region => "eu-west-1", + bucket => "logstash-test", + additional_settings => { + "force_path_style" => true, + "follow_redirects" => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +S3 bucket + +[id="{version}-plugins-{type}s-{plugin}-canned_acl"] +===== `canned_acl` + + * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`, `aws-exec-read`, `bucket-owner-read`, `bucket-owner-full-control`, `log-delivery-write` + * Default value is `"private"` + +The S3 canned ACL to use when putting the file. Defaults to "private". + +[id="{version}-plugins-{type}s-{plugin}-encoding"] +===== `encoding` + + * Value can be any of: `none`, `gzip` + * Default value is `"none"` + +Specify the content encoding. Supports ("gzip"). Defaults to "none" + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. +The endpoint should be an HTTP or HTTPS URL, e.g. https://example.com + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Specify a prefix to the uploaded filename to simulate directories on S3. +Prefix does not require leading slash. +This option supports +{logstash-ref}/event-dependent-configuration.html#sprintf[Logstash +interpolation]. For example, files can be prefixed with the event date using +`prefix = "%{+YYYY}/%{+MM}/%{+dd}"`. + +IMPORTANT: Take care when you are using interpolated strings in prefixes. This +has the potential to create large numbers of unique prefixes, causing large +numbers of in-progress uploads. This scenario may result in performance and +stability issues, which can be further exacerbated when you use a +rotation_strategy that delays uploads. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-restore"] +===== `restore` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Used to enable recovery after crash/abnormal termination. +Temporary log files will be recovered and uploaded. + +[id="{version}-plugins-{type}s-{plugin}-retry_count"] +===== `retry_count` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `Infinity` + +Allows to limit number of retries when S3 uploading fails. + +[id="{version}-plugins-{type}s-{plugin}-retry_delay"] +===== `retry_delay` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +Delay (in seconds) to wait between consecutive retries on upload failures. + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-rotation_strategy"] +===== `rotation_strategy` + + * Value can be any of: `size_and_time`, `size`, `time` + * Default value is `"size_and_time"` + +Controls when to close the file and push it to S3. + +If you set this value to `size`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>>. +If you set this value to `time`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>. +If you set this value to `size_and_time`, it uses the values from +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>> and +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>, and splits the file when +either one matches. + +The default strategy checks both size and time. The first value to +match triggers file rotation. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption"] +===== `server_side_encryption` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Specifies whether or not to use S3's server side encryption. Defaults to no encryption. + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm"] +===== `server_side_encryption_algorithm` + + * Value can be any of: `AES256`, `aws:kms` + * Default value is `"AES256"` + +Specifies what type of encryption to use when SSE is enabled. + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-signature_version"] +===== `signature_version` + + * Value can be any of: `v2`, `v4` + * There is no default value for this setting. + +The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly +specified here + +[id="{version}-plugins-{type}s-{plugin}-size_file"] +===== `size_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5242880` + +Set the file size in bytes. When the number of bytes exceeds the `size_file` +value, a new file is created. If you use tags, Logstash generates a specific size +file for every tag. + +[id="{version}-plugins-{type}s-{plugin}-ssekms_key_id"] +===== `ssekms_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key to use when specified along with server_side_encryption => aws:kms. +If server_side_encryption => aws:kms is set but this is not default KMS key is used. +http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + +[id="{version}-plugins-{type}s-{plugin}-storage_class"] +===== `storage_class` + + * Value can be any of: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA` + * Default value is `"STANDARD"` + +Specifies what S3 storage class to use when uploading the file. +More information about the different storage classes can be found: +http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html +Defaults to STANDARD. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before sending it to S3 +default to the current OS temporary directory in linux /tmp/logstash + +[id="{version}-plugins-{type}s-{plugin}-time_file"] +===== `time_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15` + +Set the time, in MINUTES, to close the current sub_time_section of bucket. +If <<{version}-plugins-{type}s-{plugin}-rotation_strategy,`rotation_strategy`>> is set to `time` or `size_and_time`, then `time_file` cannot be set to 0. +Otherwise, the plugin raises a configuration error. + +[id="{version}-plugins-{type}s-{plugin}-upload_multipart_threshold"] +===== `upload_multipart_threshold` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15728640` + +Files larger than this number are uploaded using the S3 multipart APIs + +[id="{version}-plugins-{type}s-{plugin}-upload_queue_size"] +===== `upload_queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Number of items we can keep in the local queue before uploading them + +[id="{version}-plugins-{type}s-{plugin}-upload_workers_count"] +===== `upload_workers_count` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Specify how many workers to use to upload the files to S3 + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket"] +===== `validate_credentials_on_root_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +The common use case is to define permissions on the root bucket and give Logstash +full access to write logs. +In some circumstances, you need more granular permissions on the subfolder. This +allows you to disable the check at startup. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/s3-v7.1.8.asciidoc b/docs/versioned-plugins/outputs/s3-v7.1.8.asciidoc new file mode 100644 index 000000000..2c39ea156 --- /dev/null +++ b/docs/versioned-plugins/outputs/s3-v7.1.8.asciidoc @@ -0,0 +1,453 @@ +:integration: aws +:plugin: s3 +:type: output +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin batches and uploads logstash events into Amazon Simple Storage +Service (Amazon S3). + +IMPORTANT: The S3 output plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +S3 outputs create temporary files into the OS' temporary directory. +You can specify where to save them using the `temporary_directory` option. + +IMPORTANT: For configurations containing multiple s3 outputs with the restore +option enabled, each output should define its own 'temporary_directory'. + +===== Requirements + +* Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key) +* S3 PutObject permission + +===== S3 output file + +[source,txt] +----- +`ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt` +----- + +|======= +| ls.s3 | indicates logstash plugin s3 | +| 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. | +| 2013-04-18T10.00 | represents the time whenever you specify time_file. | +| tag_hello | indicates the event's tag. | +| part0 | If you indicate size_file, it will generate more parts if your file.size > size_file. +When a file is full, it gets pushed to the bucket and then deleted from the temporary directory. +If a file is empty, it is simply deleted. Empty files will not be pushed. | +|======= + +===== Crash Recovery + +This plugin will recover and upload temporary log files after crash/abnormal termination when using `restore` set to true + +===== Usage +This is an example of logstash config: +[source,ruby] +output { + s3{ + access_key_id => "crazy_key" (optional) + secret_access_key => "monkey_access_key" (optional) + region => "eu-west-1" (optional, default = "us-east-1") + bucket => "your_bucket" (required) + size_file => 2048 (optional) - Bytes + time_file => 5 (optional) - Minutes + codec => "plain" (optional) + canned_acl => "private" (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" ) + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== S3 Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-canned_acl>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"]`|No +| <<{version}-plugins-{type}s-{plugin}-encoding>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip"]`|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-restore>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-rotation_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["size_and_time", "size", "time"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["AES256", "aws:kms"]`|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-signature_version>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["v2", "v4"]`|No +| <<{version}-plugins-{type}s-{plugin}-size_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssekms_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_class>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", "INTELLIGENT_TIERING", "GLACIER", "DEEP_ARCHIVE", "OUTPOSTS", "GLACIER_IR", "SNOW", "EXPRESS_ONEZONE"]`|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-time_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_multipart_threshold>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_workers_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +. External credentials file specified by `aws_credentials_file` +. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to S3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + output { + s3 { + access_key_id => "1234", + secret_access_key => "secret", + region => "eu-west-1", + bucket => "logstash-test", + additional_settings => { + "force_path_style" => true, + "follow_redirects" => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +S3 bucket + +[id="{version}-plugins-{type}s-{plugin}-canned_acl"] +===== `canned_acl` + + * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`, `aws-exec-read`, `bucket-owner-read`, `bucket-owner-full-control`, `log-delivery-write` + * Default value is `"private"` + +The S3 canned ACL to use when putting the file. Defaults to "private". + +[id="{version}-plugins-{type}s-{plugin}-encoding"] +===== `encoding` + + * Value can be any of: `none`, `gzip` + * Default value is `"none"` + +Specify the content encoding. Supports ("gzip"). Defaults to "none" + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. +The endpoint should be an HTTP or HTTPS URL, e.g. https://example.com + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Specify a prefix to the uploaded filename to simulate directories on S3. +Prefix does not require leading slash. +This option supports +{logstash-ref}/event-dependent-configuration.html#sprintf[Logstash +interpolation]. For example, files can be prefixed with the event date using +`prefix = "%{+YYYY}/%{+MM}/%{+dd}"`. + +IMPORTANT: Take care when you are using interpolated strings in prefixes. This +has the potential to create large numbers of unique prefixes, causing large +numbers of in-progress uploads. This scenario may result in performance and +stability issues, which can be further exacerbated when you use a +rotation_strategy that delays uploads. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-restore"] +===== `restore` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Used to enable recovery after crash/abnormal termination. +Temporary log files will be recovered and uploaded. + +[id="{version}-plugins-{type}s-{plugin}-retry_count"] +===== `retry_count` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `Infinity` + +Allows to limit number of retries when S3 uploading fails. + +[id="{version}-plugins-{type}s-{plugin}-retry_delay"] +===== `retry_delay` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +Delay (in seconds) to wait between consecutive retries on upload failures. + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-rotation_strategy"] +===== `rotation_strategy` + + * Value can be any of: `size_and_time`, `size`, `time` + * Default value is `"size_and_time"` + +Controls when to close the file and push it to S3. + +If you set this value to `size`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>>. +If you set this value to `time`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>. +If you set this value to `size_and_time`, it uses the values from +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>> and +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>, and splits the file when +either one matches. + +The default strategy checks both size and time. The first value to +match triggers file rotation. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption"] +===== `server_side_encryption` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Specifies whether or not to use S3's server side encryption. Defaults to no encryption. + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm"] +===== `server_side_encryption_algorithm` + + * Value can be any of: `AES256`, `aws:kms` + * Default value is `"AES256"` + +Specifies what type of encryption to use when SSE is enabled. + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-signature_version"] +===== `signature_version` + + * Value can be any of: `v2`, `v4` + * There is no default value for this setting. + +The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly +specified here + +[id="{version}-plugins-{type}s-{plugin}-size_file"] +===== `size_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5242880` + +Set the file size in bytes. When the number of bytes exceeds the `size_file` +value, a new file is created. If you use tags, Logstash generates a specific size +file for every tag. + +[id="{version}-plugins-{type}s-{plugin}-ssekms_key_id"] +===== `ssekms_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key to use when specified along with server_side_encryption => aws:kms. +If server_side_encryption => aws:kms is set but this is not default KMS key is used. +http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + +[id="{version}-plugins-{type}s-{plugin}-storage_class"] +===== `storage_class` + + * Value can be any of: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING`, `GLACIER`, `DEEP_ARCHIVE`, `OUTPOSTS`, `GLACIER_IR`, `SNOW`, `EXPRESS_ONEZONE` + * Default value is `"STANDARD"` + +Specifies what S3 storage class to use when uploading the file. +More information about the different storage classes can be found: +http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html +Defaults to STANDARD. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before sending it to S3 +default to the current OS temporary directory in linux /tmp/logstash + +[id="{version}-plugins-{type}s-{plugin}-time_file"] +===== `time_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15` + +Set the time, in MINUTES, to close the current sub_time_section of bucket. +If <<{version}-plugins-{type}s-{plugin}-rotation_strategy,`rotation_strategy`>> is set to `time` or `size_and_time`, then `time_file` cannot be set to 0. +Otherwise, the plugin raises a configuration error. + +[id="{version}-plugins-{type}s-{plugin}-upload_multipart_threshold"] +===== `upload_multipart_threshold` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15728640` + +Files larger than this number are uploaded using the S3 multipart APIs + +[id="{version}-plugins-{type}s-{plugin}-upload_queue_size"] +===== `upload_queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Number of items we can keep in the local queue before uploading them + +[id="{version}-plugins-{type}s-{plugin}-upload_workers_count"] +===== `upload_workers_count` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Specify how many workers to use to upload the files to S3 + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket"] +===== `validate_credentials_on_root_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +The common use case is to define permissions on the root bucket and give Logstash +full access to write logs. +In some circumstances, you need more granular permissions on the subfolder. This +allows you to disable the check at startup. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/s3-v7.2.0.asciidoc b/docs/versioned-plugins/outputs/s3-v7.2.0.asciidoc new file mode 100644 index 000000000..00a59aa74 --- /dev/null +++ b/docs/versioned-plugins/outputs/s3-v7.2.0.asciidoc @@ -0,0 +1,466 @@ +:integration: aws +:plugin: s3 +:type: output +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin batches and uploads logstash events into Amazon Simple Storage +Service (Amazon S3). + +IMPORTANT: The S3 output plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +S3 outputs create temporary files into the OS' temporary directory. +You can specify where to save them using the `temporary_directory` option. + +IMPORTANT: For configurations containing multiple s3 outputs with the restore +option enabled, each output should define its own 'temporary_directory'. + +===== Requirements + +* Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key) +* S3 PutObject permission + +===== S3 output file + +[source,txt] +----- +`ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt` +----- + +|======= +| ls.s3 | indicates logstash plugin s3 | +| 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. | +| 2013-04-18T10.00 | represents the time whenever you specify time_file. | +| tag_hello | indicates the event's tag. | +| part0 | If you indicate size_file, it will generate more parts if your file.size > size_file. +When a file is full, it gets pushed to the bucket and then deleted from the temporary directory. +If a file is empty, it is simply deleted. Empty files will not be pushed. | +|======= + +===== Crash Recovery + +This plugin will recover and upload temporary log files after crash/abnormal termination when using `restore` set to true + +===== Usage +This is an example of logstash config: +[source,ruby] +output { + s3{ + access_key_id => "crazy_key" (optional) + secret_access_key => "monkey_access_key" (optional) + region => "eu-west-1" (optional, default = "us-east-1") + bucket => "your_bucket" (required) + size_file => 2048 (optional) - Bytes + time_file => 5 (optional) - Minutes + codec => "plain" (optional) + canned_acl => "private" (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" ) + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== S3 Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-canned_acl>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"]`|No +| <<{version}-plugins-{type}s-{plugin}-encoding>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip"]`|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-restore>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-rotation_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["size_and_time", "size", "time"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["AES256", "aws:kms"]`|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-signature_version>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["v2", "v4"]`|No +| <<{version}-plugins-{type}s-{plugin}-size_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssekms_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_class>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", "INTELLIGENT_TIERING", "GLACIER", "DEEP_ARCHIVE", "OUTPOSTS", "GLACIER_IR", "SNOW", "EXPRESS_ONEZONE"]`|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-time_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_multipart_threshold>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_workers_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +. External credentials file specified by `aws_credentials_file` +. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to S3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + output { + s3 { + access_key_id => "1234", + secret_access_key => "secret", + region => "eu-west-1", + bucket => "logstash-test", + additional_settings => { + "force_path_style" => true, + "follow_redirects" => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +S3 bucket + +[id="{version}-plugins-{type}s-{plugin}-canned_acl"] +===== `canned_acl` + + * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`, `aws-exec-read`, `bucket-owner-read`, `bucket-owner-full-control`, `log-delivery-write` + * Default value is `"private"` + +The S3 canned ACL to use when putting the file. Defaults to "private". + +[id="{version}-plugins-{type}s-{plugin}-encoding"] +===== `encoding` + + * Value can be any of: `none`, `gzip` + * Default value is `"none"` + +Specify the content encoding. Supports ("gzip"). Defaults to "none" + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. +The endpoint should be an HTTP or HTTPS URL, e.g. https://example.com + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Specify a prefix to the uploaded filename to simulate directories on S3. +Prefix does not require leading slash. +This option supports +{logstash-ref}/event-dependent-configuration.html#sprintf[Logstash +interpolation]. For example, files can be prefixed with the event date using +`prefix = "%{+YYYY}/%{+MM}/%{+dd}"`. + +IMPORTANT: Take care when you are using interpolated strings in prefixes. This +has the potential to create large numbers of unique prefixes, causing large +numbers of in-progress uploads. This scenario may result in performance and +stability issues, which can be further exacerbated when you use a +rotation_strategy that delays uploads. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-restore"] +===== `restore` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Used to enable recovery after crash/abnormal termination. +Temporary log files will be recovered and uploaded. + +[id="{version}-plugins-{type}s-{plugin}-retry_count"] +===== `retry_count` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `Infinity` + +Allows to limit number of retries when S3 uploading fails. + +[id="{version}-plugins-{type}s-{plugin}-retry_delay"] +===== `retry_delay` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +Delay (in seconds) to wait between consecutive retries on upload failures. + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-rotation_strategy"] +===== `rotation_strategy` + + * Value can be any of: `size_and_time`, `size`, `time` + * Default value is `"size_and_time"` + +Controls when to close the file and push it to S3. + +If you set this value to `size`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>>. +If you set this value to `time`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>. +If you set this value to `size_and_time`, it uses the values from +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>> and +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>, and splits the file when +either one matches. + +The default strategy checks both size and time. The first value to +match triggers file rotation. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption"] +===== `server_side_encryption` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Specifies whether or not to use S3's server side encryption. Defaults to no encryption. + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm"] +===== `server_side_encryption_algorithm` + + * Value can be any of: `AES256`, `aws:kms` + * Default value is `"AES256"` + +Specifies what type of encryption to use when SSE is enabled. + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-signature_version"] +===== `signature_version` + + * Value can be any of: `v2`, `v4` + * There is no default value for this setting. + +The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly +specified here + +[id="{version}-plugins-{type}s-{plugin}-size_file"] +===== `size_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5242880` + +Set the file size in bytes. When the number of bytes exceeds the `size_file` +value, a new file is created. If you use tags, Logstash generates a specific size +file for every tag. + +[id="{version}-plugins-{type}s-{plugin}-ssekms_key_id"] +===== `ssekms_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key to use when specified along with server_side_encryption => aws:kms. +If server_side_encryption => aws:kms is set but this is not default KMS key is used. +http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + +[id="{version}-plugins-{type}s-{plugin}-storage_class"] +===== `storage_class` + + * Value can be any of: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING`, `GLACIER`, `DEEP_ARCHIVE`, `OUTPOSTS`, `GLACIER_IR`, `SNOW`, `EXPRESS_ONEZONE` + * Default value is `"STANDARD"` + +Specifies what S3 storage class to use when uploading the file. +More information about the different storage classes can be found: +http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html +Defaults to STANDARD. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before sending it to S3 +default to the current OS temporary directory in linux /tmp/logstash + +[id="{version}-plugins-{type}s-{plugin}-time_file"] +===== `time_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15` + +Set the time, in MINUTES, to close the current sub_time_section of bucket. +If <<{version}-plugins-{type}s-{plugin}-rotation_strategy,`rotation_strategy`>> is set to `time` or `size_and_time`, then `time_file` cannot be set to 0. +Otherwise, the plugin raises a configuration error. + +[id="{version}-plugins-{type}s-{plugin}-upload_multipart_threshold"] +===== `upload_multipart_threshold` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15728640` + +Files larger than this number are uploaded using the S3 multipart APIs + +[id="{version}-plugins-{type}s-{plugin}-upload_queue_size"] +===== `upload_queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Number of items we can keep in the local queue before uploading them + +[id="{version}-plugins-{type}s-{plugin}-upload_workers_count"] +===== `upload_workers_count` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Specify how many workers to use to upload the files to S3 + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket"] +===== `validate_credentials_on_root_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +The common use case is to define permissions on the root bucket and give Logstash +full access to write logs. +In some circumstances, you need more granular permissions on the subfolder. This +allows you to disable the check at startup. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/s3-v7.2.1.asciidoc b/docs/versioned-plugins/outputs/s3-v7.2.1.asciidoc new file mode 100644 index 000000000..f970e34d9 --- /dev/null +++ b/docs/versioned-plugins/outputs/s3-v7.2.1.asciidoc @@ -0,0 +1,466 @@ +:integration: aws +:plugin: s3 +:type: output +:default_codec: line + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== S3 output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin batches and uploads logstash events into Amazon Simple Storage +Service (Amazon S3). + +IMPORTANT: The S3 output plugin only supports AWS S3. +Other S3 compatible storage solutions are not supported. + +S3 outputs create temporary files into the OS' temporary directory. +You can specify where to save them using the `temporary_directory` option. + +IMPORTANT: For configurations containing multiple s3 outputs with the restore +option enabled, each output should define its own 'temporary_directory'. + +===== Requirements + +* Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key) +* S3 PutObject permission + +===== S3 output file + +[source,txt] +----- +`ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt` +----- + +|======= +| ls.s3 | indicates logstash plugin s3 | +| 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. | +| 2013-04-18T10.00 | represents the time whenever you specify time_file. | +| tag_hello | indicates the event's tag. | +| part0 | If you indicate size_file, it will generate more parts if your file.size > size_file. +When a file is full, it gets pushed to the bucket and then deleted from the temporary directory. +If a file is empty, it is simply deleted. Empty files will not be pushed. | +|======= + +===== Crash Recovery + +This plugin will recover and upload temporary log files after crash/abnormal termination when using `restore` set to true + +===== Usage +This is an example of logstash config: +[source,ruby] +output { + s3{ + access_key_id => "crazy_key" (optional) + secret_access_key => "monkey_access_key" (optional) + region => "eu-west-1" (optional, default = "us-east-1") + bucket => "your_bucket" (required) + size_file => 2048 (optional) - Bytes + time_file => 5 (optional) - Minutes + codec => "plain" (optional) + canned_acl => "private" (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" ) + } + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== S3 Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-additional_settings>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-bucket>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-canned_acl>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"]`|No +| <<{version}-plugins-{type}s-{plugin}-encoding>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "gzip"]`|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prefix>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-restore>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-retry_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_delay>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-rotation_strategy>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["size_and_time", "size", "time"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["AES256", "aws:kms"]`|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-signature_version>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["v2", "v4"]`|No +| <<{version}-plugins-{type}s-{plugin}-size_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssekms_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-storage_class>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", "INTELLIGENT_TIERING", "GLACIER", "DEEP_ARCHIVE", "OUTPOSTS", "GLACIER_IR", "SNOW", "EXPRESS_ONEZONE"]`|No +| <<{version}-plugins-{type}s-{plugin}-temporary_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-time_file>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_multipart_threshold>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_queue_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-upload_workers_count>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +. External credentials file specified by `aws_credentials_file` +. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-additional_settings"] +===== `additional_settings` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Key-value pairs of settings and corresponding values used to parametrize +the connection to S3. See full list in https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html[the AWS SDK documentation]. Example: + +[source,ruby] + output { + s3 { + access_key_id => "1234", + secret_access_key => "secret", + region => "eu-west-1", + bucket => "logstash-test", + additional_settings => { + "force_path_style" => true, + "follow_redirects" => false + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-bucket"] +===== `bucket` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +S3 bucket + +[id="{version}-plugins-{type}s-{plugin}-canned_acl"] +===== `canned_acl` + + * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`, `aws-exec-read`, `bucket-owner-read`, `bucket-owner-full-control`, `log-delivery-write` + * Default value is `"private"` + +The S3 canned ACL to use when putting the file. Defaults to "private". + +[id="{version}-plugins-{type}s-{plugin}-encoding"] +===== `encoding` + + * Value can be any of: `none`, `gzip` + * Default value is `"none"` + +Specify the content encoding. Supports ("gzip"). Defaults to "none" + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. +The endpoint should be an HTTP or HTTPS URL, e.g. https://example.com + +[id="{version}-plugins-{type}s-{plugin}-prefix"] +===== `prefix` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Specify a prefix to the uploaded filename to simulate directories on S3. +Prefix does not require leading slash. +This option supports +{logstash-ref}/event-dependent-configuration.html#sprintf[Logstash +interpolation]. For example, files can be prefixed with the event date using +`prefix = "%{+YYYY}/%{+MM}/%{+dd}"`. + +IMPORTANT: Take care when you are using interpolated strings in prefixes. This +has the potential to create large numbers of unique prefixes, causing large +numbers of in-progress uploads. This scenario may result in performance and +stability issues, which can be further exacerbated when you use a +rotation_strategy that delays uploads. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-restore"] +===== `restore` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Used to enable recovery after crash/abnormal termination. +Temporary log files will be recovered and uploaded. + +[id="{version}-plugins-{type}s-{plugin}-retry_count"] +===== `retry_count` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `Infinity` + +Allows to limit number of retries when S3 uploading fails. + +[id="{version}-plugins-{type}s-{plugin}-retry_delay"] +===== `retry_delay` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +Delay (in seconds) to wait between consecutive retries on upload failures. + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-rotation_strategy"] +===== `rotation_strategy` + + * Value can be any of: `size_and_time`, `size`, `time` + * Default value is `"size_and_time"` + +Controls when to close the file and push it to S3. + +If you set this value to `size`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>>. +If you set this value to `time`, it uses the value set in +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>. +If you set this value to `size_and_time`, it uses the values from +<<{version}-plugins-{type}s-{plugin}-size_file,`size_file`>> and +<<{version}-plugins-{type}s-{plugin}-time_file,`time_file`>>, and splits the file when +either one matches. + +The default strategy checks both size and time. The first value to +match triggers file rotation. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption"] +===== `server_side_encryption` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Specifies whether or not to use S3's server side encryption. Defaults to no encryption. + +[id="{version}-plugins-{type}s-{plugin}-server_side_encryption_algorithm"] +===== `server_side_encryption_algorithm` + + * Value can be any of: `AES256`, `aws:kms` + * Default value is `"AES256"` + +Specifies what type of encryption to use when SSE is enabled. + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-signature_version"] +===== `signature_version` + + * Value can be any of: `v2`, `v4` + * There is no default value for this setting. + +The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly +specified here + +[id="{version}-plugins-{type}s-{plugin}-size_file"] +===== `size_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5242880` + +Set the file size in bytes. When the number of bytes exceeds the `size_file` +value, a new file is created. If you use tags, Logstash generates a specific size +file for every tag. + +[id="{version}-plugins-{type}s-{plugin}-ssekms_key_id"] +===== `ssekms_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The key to use when specified along with server_side_encryption => aws:kms. +If server_side_encryption => aws:kms is set but this is not default KMS key is used. +http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + +[id="{version}-plugins-{type}s-{plugin}-storage_class"] +===== `storage_class` + + * Value can be any of: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING`, `GLACIER`, `DEEP_ARCHIVE`, `OUTPOSTS`, `GLACIER_IR`, `SNOW`, `EXPRESS_ONEZONE` + * Default value is `"STANDARD"` + +Specifies what S3 storage class to use when uploading the file. +More information about the different storage classes can be found: +http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html +Defaults to STANDARD. + +[id="{version}-plugins-{type}s-{plugin}-temporary_directory"] +===== `temporary_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/tmp/logstash"` + +Set the directory where logstash will store the tmp files before sending it to S3 +default to the current OS temporary directory in linux /tmp/logstash + +[id="{version}-plugins-{type}s-{plugin}-time_file"] +===== `time_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15` + +Set the time, in MINUTES, to close the current sub_time_section of bucket. +If <<{version}-plugins-{type}s-{plugin}-rotation_strategy,`rotation_strategy`>> is set to `time` or `size_and_time`, then `time_file` cannot be set to 0. +Otherwise, the plugin raises a configuration error. + +[id="{version}-plugins-{type}s-{plugin}-upload_multipart_threshold"] +===== `upload_multipart_threshold` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `15728640` + +Files larger than this number are uploaded using the S3 multipart APIs + +[id="{version}-plugins-{type}s-{plugin}-upload_queue_size"] +===== `upload_queue_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Number of items we can keep in the local queue before uploading them + +[id="{version}-plugins-{type}s-{plugin}-upload_workers_count"] +===== `upload_workers_count` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `4` + +Specify how many workers to use to upload the files to S3 + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-validate_credentials_on_root_bucket"] +===== `validate_credentials_on_root_bucket` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +The common use case is to define permissions on the root bucket and give Logstash +full access to write logs. +In some circumstances, you need more granular permissions on the subfolder. This +allows you to disable the check at startup. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/sns-index.asciidoc b/docs/versioned-plugins/outputs/sns-index.asciidoc index 3d99daa35..d2e476159 100644 --- a/docs/versioned-plugins/outputs/sns-index.asciidoc +++ b/docs/versioned-plugins/outputs/sns-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -21,6 +25,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::sns-v7.2.1.asciidoc[] +include::sns-v7.2.0.asciidoc[] +include::sns-v7.1.8.asciidoc[] +include::sns-v7.1.7.asciidoc[] include::sns-v7.1.6.asciidoc[] include::sns-v7.1.5.asciidoc[] include::sns-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/outputs/sns-v7.1.7.asciidoc b/docs/versioned-plugins/outputs/sns-v7.1.7.asciidoc new file mode 100644 index 000000000..e452785e0 --- /dev/null +++ b/docs/versioned-plugins/outputs/sns-v7.1.7.asciidoc @@ -0,0 +1,176 @@ +:integration: aws +:plugin: sns +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sns output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +SNS output. + +Send events to Amazon's Simple Notification Service, a hosted pub/sub +framework. It supports various subscription types, including email, HTTP/S, SMS, and SQS. + +For further documentation about the service see: + + http://docs.amazonwebservices.com/sns/latest/api/ + +This plugin looks for the following fields on events it receives: + + * `sns` - If no ARN is found in the configuration file, this will be used as + the ARN to publish. + * `sns_subject` - The subject line that should be used. + Optional. The "%{host}" will be used if `sns_subject` is not present. The subject + will be truncated to 100 characters. If `sns_subject` is set to a non-string value a JSON version of that value will be saved. + * `sns_message` - Optional string of message to be sent. If this is set to a non-string value it will be encoded with the specified `codec`. If this is not set the entire event will be encoded with the codec. + with the @message truncated so that the length of the JSON fits in + `32768` bytes. + +==== Upgrading to 2.0.0 + +This plugin used to have a `format` option for controlling the encoding of messages prior to being sent to SNS. +This plugin now uses the logstash standard {logstash-ref}/configuration-file-structure.html#codec[codec] option for encoding instead. +If you want the same 'plain' format as the v0/1 codec (`format => "plain"`) use `codec => "s3_plain"`. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sns Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-publish_boot_message_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-arn"] +===== `arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Optional ARN to send messages to. If you do not set this you must +include the `sns` field in your events to set the ARN on a per-message basis! + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-publish_boot_message_arn"] +===== `publish_boot_message_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When an ARN for an SNS topic is specified here, the message +"Logstash successfully booted" will be sent to it when this plugin +is registered. + +Example: arn:aws:sns:us-east-1:770975001275:logstash-testing + + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/sns-v7.1.8.asciidoc b/docs/versioned-plugins/outputs/sns-v7.1.8.asciidoc new file mode 100644 index 000000000..08885e3f2 --- /dev/null +++ b/docs/versioned-plugins/outputs/sns-v7.1.8.asciidoc @@ -0,0 +1,176 @@ +:integration: aws +:plugin: sns +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sns output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +SNS output. + +Send events to Amazon's Simple Notification Service, a hosted pub/sub +framework. It supports various subscription types, including email, HTTP/S, SMS, and SQS. + +For further documentation about the service see: + + http://docs.amazonwebservices.com/sns/latest/api/ + +This plugin looks for the following fields on events it receives: + + * `sns` - If no ARN is found in the configuration file, this will be used as + the ARN to publish. + * `sns_subject` - The subject line that should be used. + Optional. The "%{host}" will be used if `sns_subject` is not present. The subject + will be truncated to 100 characters. If `sns_subject` is set to a non-string value a JSON version of that value will be saved. + * `sns_message` - Optional string of message to be sent. If this is set to a non-string value it will be encoded with the specified `codec`. If this is not set the entire event will be encoded with the codec. + with the @message truncated so that the length of the JSON fits in + `32768` bytes. + +==== Upgrading to 2.0.0 + +This plugin used to have a `format` option for controlling the encoding of messages prior to being sent to SNS. +This plugin now uses the logstash standard {logstash-ref}/configuration-file-structure.html#codec[codec] option for encoding instead. +If you want the same 'plain' format as the v0/1 codec (`format => "plain"`) use `codec => "s3_plain"`. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sns Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-publish_boot_message_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-arn"] +===== `arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Optional ARN to send messages to. If you do not set this you must +include the `sns` field in your events to set the ARN on a per-message basis! + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-publish_boot_message_arn"] +===== `publish_boot_message_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When an ARN for an SNS topic is specified here, the message +"Logstash successfully booted" will be sent to it when this plugin +is registered. + +Example: arn:aws:sns:us-east-1:770975001275:logstash-testing + + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/sns-v7.2.0.asciidoc b/docs/versioned-plugins/outputs/sns-v7.2.0.asciidoc new file mode 100644 index 000000000..3e8a97157 --- /dev/null +++ b/docs/versioned-plugins/outputs/sns-v7.2.0.asciidoc @@ -0,0 +1,188 @@ +:integration: aws +:plugin: sns +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sns output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +SNS output. + +Send events to Amazon's Simple Notification Service, a hosted pub/sub +framework. It supports various subscription types, including email, HTTP/S, SMS, and SQS. + +For further documentation about the service see: + + http://docs.amazonwebservices.com/sns/latest/api/ + +This plugin looks for the following fields on events it receives: + + * `sns` - If no ARN is found in the configuration file, this will be used as + the ARN to publish. + * `sns_subject` - The subject line that should be used. + Optional. The "%{host}" will be used if `sns_subject` is not present. The subject + will be truncated to 100 characters. If `sns_subject` is set to a non-string value a JSON version of that value will be saved. + * `sns_message` - Optional string of message to be sent. If this is set to a non-string value it will be encoded with the specified `codec`. If this is not set the entire event will be encoded with the codec. + with the @message truncated so that the length of the JSON fits in + `32768` bytes. + +==== Upgrading to 2.0.0 + +This plugin used to have a `format` option for controlling the encoding of messages prior to being sent to SNS. +This plugin now uses the logstash standard {logstash-ref}/configuration-file-structure.html#codec[codec] option for encoding instead. +If you want the same 'plain' format as the v0/1 codec (`format => "plain"`) use `codec => "s3_plain"`. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sns Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-publish_boot_message_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-arn"] +===== `arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Optional ARN to send messages to. If you do not set this you must +include the `sns` field in your events to set the ARN on a per-message basis! + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-publish_boot_message_arn"] +===== `publish_boot_message_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When an ARN for an SNS topic is specified here, the message +"Logstash successfully booted" will be sent to it when this plugin +is registered. + +Example: arn:aws:sns:us-east-1:770975001275:logstash-testing + + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/sns-v7.2.1.asciidoc b/docs/versioned-plugins/outputs/sns-v7.2.1.asciidoc new file mode 100644 index 000000000..7fa1df175 --- /dev/null +++ b/docs/versioned-plugins/outputs/sns-v7.2.1.asciidoc @@ -0,0 +1,176 @@ +:integration: aws +:plugin: sns +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sns output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +SNS output. + +Send events to Amazon's Simple Notification Service, a hosted pub/sub +framework. It supports various subscription types, including email, HTTP/S, SMS, and SQS. + +For further documentation about the service see: + + http://docs.amazonwebservices.com/sns/latest/api/ + +This plugin looks for the following fields on events it receives: + + * `sns` - If no ARN is found in the configuration file, this will be used as + the ARN to publish. + * `sns_subject` - The subject line that should be used. + Optional. The "%{host}" will be used if `sns_subject` is not present. The subject + will be truncated to 100 characters. If `sns_subject` is set to a non-string value a JSON version of that value will be saved. + * `sns_message` - Optional string of message to be sent. If this is set to a non-string value it will be encoded with the specified `codec`. If this is not set the entire event will be encoded with the codec. + with the @message truncated so that the length of the JSON fits in + `32768` bytes. + +==== Upgrading to 2.0.0 + +This plugin used to have a `format` option for controlling the encoding of messages prior to being sent to SNS. +This plugin now uses the logstash standard {logstash-ref}/configuration-file-structure.html#codec[codec] option for encoding instead. +If you want the same 'plain' format as the v0/1 codec (`format => "plain"`) use `codec => "s3_plain"`. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sns Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-publish_boot_message_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-arn"] +===== `arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Optional ARN to send messages to. If you do not set this you must +include the `sns` field in your events to set the ARN on a per-message basis! + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-publish_boot_message_arn"] +===== `publish_boot_message_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When an ARN for an SNS topic is specified here, the message +"Logstash successfully booted" will be sent to it when this plugin +is registered. + +Example: arn:aws:sns:us-east-1:770975001275:logstash-testing + + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1` + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/sqs-index.asciidoc b/docs/versioned-plugins/outputs/sqs-index.asciidoc index 11a529569..c61de2f3f 100644 --- a/docs/versioned-plugins/outputs/sqs-index.asciidoc +++ b/docs/versioned-plugins/outputs/sqs-index.asciidoc @@ -5,6 +5,10 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-02-13 +| <> | 2025-02-10 +| <> | 2024-07-26 +| <> | 2024-07-02 | <> | 2023-09-12 | <> | 2023-08-04 | <> | 2023-06-15 @@ -25,6 +29,10 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::sqs-v7.2.1.asciidoc[] +include::sqs-v7.2.0.asciidoc[] +include::sqs-v7.1.8.asciidoc[] +include::sqs-v7.1.7.asciidoc[] include::sqs-v7.1.6.asciidoc[] include::sqs-v7.1.5.asciidoc[] include::sqs-v7.1.4.asciidoc[] diff --git a/docs/versioned-plugins/outputs/sqs-v7.1.7.asciidoc b/docs/versioned-plugins/outputs/sqs-v7.1.7.asciidoc new file mode 100644 index 000000000..cbf32d8a5 --- /dev/null +++ b/docs/versioned-plugins/outputs/sqs-v7.1.7.asciidoc @@ -0,0 +1,251 @@ +:integration: aws +:plugin: sqs +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.7 +:release_date: 2024-07-02 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.7/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the Amazon Web +Services suite of tools. Although SQS is similar to other queuing systems +such as Advanced Message Queuing Protocol (AMQP), it uses a custom API and +requires that you have an AWS account. See http://aws.amazon.com/sqs/ for +more details on how SQS works, what the pricing schedule looks like and how +to setup a queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:GetQueueUrl` + * `sqs:SendMessage` + * `sqs:SendMessageBatch` + +Typically, you should setup an IAM policy, create a user and apply the IAM +policy to the user. See http://aws.amazon.com/iam/ for more details on +setting up AWS identities. A sample policy is as follows: + +[source,json] +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sqs:GetQueueUrl", + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "Resource": "arn:aws:sqs:us-east-1:123456789012:my-sqs-queue" + } + ] +} + +==== Batch Publishing +This output publishes messages to SQS in batches in order to optimize event +throughput and increase performance. This is done using the +[`SendMessageBatch`](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html) +API. When publishing messages to SQS in batches, the following service limits +must be respected (see +[Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)): + + * The maximum allowed individual message size is 256KiB. + * The maximum total payload size (i.e. the sum of the sizes of all + individual messages within a batch) is also 256KiB. + +This plugin will dynamically adjust the size of the batch published to SQS in +order to ensure that the total payload size does not exceed 256KiB. + +WARNING: This output cannot currently handle messages larger than 256KiB. Any +single message exceeding this size will be dropped. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_events>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-message_max_size>> |{logstash-ref}/configuration-file-structure.html#bytes[bytes]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_events"] +===== `batch_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The number of events to be sent in each batch. Set this to `1` to disable +the batch sending of messages. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-message_max_size"] +===== `message_max_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#bytes[bytes] + * Default value is `"256KiB"` + +The maximum number of bytes for any message sent to SQS. Messages exceeding +this size will be dropped. See +http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the target SQS queue. Note that this is just the name of the +queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The owning account id of the target SQS queue. IAM permissions need to be +configured on both accounts to function. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/sqs-v7.1.8.asciidoc b/docs/versioned-plugins/outputs/sqs-v7.1.8.asciidoc new file mode 100644 index 000000000..0d021d57b --- /dev/null +++ b/docs/versioned-plugins/outputs/sqs-v7.1.8.asciidoc @@ -0,0 +1,251 @@ +:integration: aws +:plugin: sqs +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.1.8 +:release_date: 2024-07-26 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.1.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the Amazon Web +Services suite of tools. Although SQS is similar to other queuing systems +such as Advanced Message Queuing Protocol (AMQP), it uses a custom API and +requires that you have an AWS account. See http://aws.amazon.com/sqs/ for +more details on how SQS works, what the pricing schedule looks like and how +to setup a queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:GetQueueUrl` + * `sqs:SendMessage` + * `sqs:SendMessageBatch` + +Typically, you should setup an IAM policy, create a user and apply the IAM +policy to the user. See http://aws.amazon.com/iam/ for more details on +setting up AWS identities. A sample policy is as follows: + +[source,json] +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sqs:GetQueueUrl", + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "Resource": "arn:aws:sqs:us-east-1:123456789012:my-sqs-queue" + } + ] +} + +==== Batch Publishing +This output publishes messages to SQS in batches in order to optimize event +throughput and increase performance. This is done using the +[`SendMessageBatch`](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html) +API. When publishing messages to SQS in batches, the following service limits +must be respected (see +[Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)): + + * The maximum allowed individual message size is 256KiB. + * The maximum total payload size (i.e. the sum of the sizes of all + individual messages within a batch) is also 256KiB. + +This plugin will dynamically adjust the size of the batch published to SQS in +order to ensure that the total payload size does not exceed 256KiB. + +WARNING: This output cannot currently handle messages larger than 256KiB. Any +single message exceeding this size will be dropped. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_events>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-message_max_size>> |{logstash-ref}/configuration-file-structure.html#bytes[bytes]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_events"] +===== `batch_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The number of events to be sent in each batch. Set this to `1` to disable +the batch sending of messages. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-message_max_size"] +===== `message_max_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#bytes[bytes] + * Default value is `"256KiB"` + +The maximum number of bytes for any message sent to SQS. Messages exceeding +this size will be dropped. See +http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the target SQS queue. Note that this is just the name of the +queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The owning account id of the target SQS queue. IAM permissions need to be +configured on both accounts to function. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/sqs-v7.2.0.asciidoc b/docs/versioned-plugins/outputs/sqs-v7.2.0.asciidoc new file mode 100644 index 000000000..e6240e979 --- /dev/null +++ b/docs/versioned-plugins/outputs/sqs-v7.2.0.asciidoc @@ -0,0 +1,264 @@ +:integration: aws +:plugin: sqs +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.0 +:release_date: 2025-02-10 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the Amazon Web +Services suite of tools. Although SQS is similar to other queuing systems +such as Advanced Message Queuing Protocol (AMQP), it uses a custom API and +requires that you have an AWS account. See http://aws.amazon.com/sqs/ for +more details on how SQS works, what the pricing schedule looks like and how +to setup a queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:GetQueueUrl` + * `sqs:SendMessage` + * `sqs:SendMessageBatch` + +Typically, you should setup an IAM policy, create a user and apply the IAM +policy to the user. See http://aws.amazon.com/iam/ for more details on +setting up AWS identities. A sample policy is as follows: + +[source,json] +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sqs:GetQueueUrl", + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "Resource": "arn:aws:sqs:us-east-1:123456789012:my-sqs-queue" + } + ] +} + +==== Batch Publishing +This output publishes messages to SQS in batches in order to optimize event +throughput and increase performance. This is done using the +[`SendMessageBatch`](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html) +API. When publishing messages to SQS in batches, the following service limits +must be respected (see +[Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)): + + * The maximum allowed individual message size is 256KiB. + * The maximum total payload size (i.e. the sum of the sizes of all + individual messages within a batch) is also 256KiB. + +This plugin will dynamically adjust the size of the batch published to SQS in +order to ensure that the total payload size does not exceed 256KiB. + +WARNING: This output cannot currently handle messages larger than 256KiB. Any +single message exceeding this size will be dropped. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_events>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-message_max_size>> |{logstash-ref}/configuration-file-structure.html#bytes[bytes]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_events"] +===== `batch_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The number of events to be sent in each batch. Set this to `1` to disable +the batch sending of messages. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-message_max_size"] +===== `message_max_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#bytes[bytes] + * Default value is `"256KiB"` + +The maximum number of bytes for any message sent to SQS. Messages exceeding +this size will be dropped. See +http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the target SQS queue. Note that this is just the name of the +queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The owning account id of the target SQS queue. IAM permissions need to be +configured on both accounts to function. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/sqs-v7.2.1.asciidoc b/docs/versioned-plugins/outputs/sqs-v7.2.1.asciidoc new file mode 100644 index 000000000..84db0931e --- /dev/null +++ b/docs/versioned-plugins/outputs/sqs-v7.2.1.asciidoc @@ -0,0 +1,264 @@ +:integration: aws +:plugin: sqs +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.2.1 +:release_date: 2025-02-13 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-aws/blob/v7.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Sqs output plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the Amazon Web +Services suite of tools. Although SQS is similar to other queuing systems +such as Advanced Message Queuing Protocol (AMQP), it uses a custom API and +requires that you have an AWS account. See http://aws.amazon.com/sqs/ for +more details on how SQS works, what the pricing schedule looks like and how +to setup a queue. + +The "consumer" identity must have the following permissions on the queue: + + * `sqs:GetQueueUrl` + * `sqs:SendMessage` + * `sqs:SendMessageBatch` + +Typically, you should setup an IAM policy, create a user and apply the IAM +policy to the user. See http://aws.amazon.com/iam/ for more details on +setting up AWS identities. A sample policy is as follows: + +[source,json] +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sqs:GetQueueUrl", + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "Resource": "arn:aws:sqs:us-east-1:123456789012:my-sqs-queue" + } + ] +} + +==== Batch Publishing +This output publishes messages to SQS in batches in order to optimize event +throughput and increase performance. This is done using the +[`SendMessageBatch`](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html) +API. When publishing messages to SQS in batches, the following service limits +must be respected (see +[Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)): + + * The maximum allowed individual message size is 256KiB. + * The maximum total payload size (i.e. the sum of the sizes of all + individual messages within a batch) is also 256KiB. + +This plugin will dynamically adjust the size of the batch published to SQS in +order to ensure that the total payload size does not exceed 256KiB. + +WARNING: This output cannot currently handle messages larger than 256KiB. Any +single message exceeding this size will be dropped. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Sqs Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-access_key_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-aws_credentials_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-batch_events>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-endpoint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-message_max_size>> |{logstash-ref}/configuration-file-structure.html#bytes[bytes]|No +| <<{version}-plugins-{type}s-{plugin}-proxy_uri>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-queue>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-region>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_arn>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-role_session_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-secret_access_key>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-session_token>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-web_identity_token_file>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-access_key_id"] +===== `access_key_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + +1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config +2. External credentials file specified by `aws_credentials_file` +3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` +5. IAM Instance Profile (available when running inside EC2) + +[id="{version}-plugins-{type}s-{plugin}-aws_credentials_file"] +===== `aws_credentials_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Path to YAML file containing a hash of AWS credentials. +This file will only be loaded if `access_key_id` and +`secret_access_key` aren't set. The contents of the +file should look like this: + +[source,ruby] +---------------------------------- + :access_key_id: "12345" + :secret_access_key: "54321" +---------------------------------- + + +[id="{version}-plugins-{type}s-{plugin}-batch_events"] +===== `batch_events` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +The number of events to be sent in each batch. Set this to `1` to disable +the batch sending of messages. + +[id="{version}-plugins-{type}s-{plugin}-endpoint"] +===== `endpoint` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The endpoint to connect to. By default it is constructed using the value of `region`. +This is useful when connecting to S3 compatible services, but beware that these aren't +guaranteed to work correctly with the AWS SDK. + +[id="{version}-plugins-{type}s-{plugin}-message_max_size"] +===== `message_max_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#bytes[bytes] + * Default value is `"256KiB"` + +The maximum number of bytes for any message sent to SQS. Messages exceeding +this size will be dropped. See +http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html. + +[id="{version}-plugins-{type}s-{plugin}-proxy_uri"] +===== `proxy_uri` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +URI to proxy server if required + +[id="{version}-plugins-{type}s-{plugin}-queue"] +===== `queue` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The name of the target SQS queue. Note that this is just the name of the +queue, not the URL or ARN. + +[id="{version}-plugins-{type}s-{plugin}-queue_owner_aws_account_id"] +===== `queue_owner_aws_account_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The owning account id of the target SQS queue. IAM permissions need to be +configured on both accounts to function. + +[id="{version}-plugins-{type}s-{plugin}-region"] +===== `region` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"us-east-1"` + +The AWS Region + +[id="{version}-plugins-{type}s-{plugin}-role_arn"] +===== `role_arn` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS IAM Role to assume, if any. +This is used to generate temporary credentials, typically for cross-account access. +See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information. + +[id="{version}-plugins-{type}s-{plugin}-role_session_name"] +===== `role_session_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"logstash"` + +Session name to use when assuming an IAM role. + +[id="{version}-plugins-{type}s-{plugin}-secret_access_key"] +===== `secret_access_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Secret Access Key + +[id="{version}-plugins-{type}s-{plugin}-session_token"] +===== `session_token` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The AWS Session token for temporary credential + +[id="{version}-plugins-{type}s-{plugin}-use_aws_bundled_ca"] +===== `use_aws_bundled_ca` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* Default value is `false` + +Use bundled CA certificates that ship with AWS SDK to verify SSL peer certificates. +For cases where the default certificates are unavailable, e.g. Windows, +you can set this to `true`. + +[id="{version}-plugins-{type}s-{plugin}-web_identity_token_file"] +===== `web_identity_token_file` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] + +Absolute path to the file on disk containing OIDC token for https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html[IRSA]. +Set the path to `"/var/run/secrets/eks.amazonaws.com/serviceaccount/token"` when using Amazon EKS. +If specified, <<{version}-plugins-{type}s-{plugin}-role_arn>> must not be empty. + +For the details of setting up IAM roles and Kubernetes service accounts, check out https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html[userguide] + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/outputs/tcp-index.asciidoc b/docs/versioned-plugins/outputs/tcp-index.asciidoc index a2210e5b7..a5510ba41 100644 --- a/docs/versioned-plugins/outputs/tcp-index.asciidoc +++ b/docs/versioned-plugins/outputs/tcp-index.asciidoc @@ -5,6 +5,11 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-04-29 +| <> | 2025-01-10 +| <> | 2025-04-29 +| <> | 2024-06-05 +| <> | 2023-12-19 | <> | 2023-06-23 | <> | 2022-08-21 | <> | 2022-06-06 @@ -21,6 +26,11 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::tcp-v7.0.1.asciidoc[] +include::tcp-v7.0.0.asciidoc[] +include::tcp-v6.2.2.asciidoc[] +include::tcp-v6.2.1.asciidoc[] +include::tcp-v6.2.0.asciidoc[] include::tcp-v6.1.2.asciidoc[] include::tcp-v6.1.1.asciidoc[] include::tcp-v6.1.0.asciidoc[] diff --git a/docs/versioned-plugins/outputs/tcp-v6.2.0.asciidoc b/docs/versioned-plugins/outputs/tcp-v6.2.0.asciidoc new file mode 100644 index 000000000..aa31e2242 --- /dev/null +++ b/docs/versioned-plugins/outputs/tcp-v6.2.0.asciidoc @@ -0,0 +1,239 @@ +:plugin: tcp +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.2.0 +:release_date: 2023-12-19 +:changelog_url: https://github.com/logstash-plugins/logstash-output-tcp/blob/v6.2.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Write events over a TCP socket. + +Each event json is separated by a newline. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-reconnect_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"client"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +When connect failed,retry interval in sec. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cacert"] +===== `ssl_cacert` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The SSL CA certificate, chainfile or CA path. The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate path + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `none` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key path + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/tcp-v6.2.1.asciidoc b/docs/versioned-plugins/outputs/tcp-v6.2.1.asciidoc new file mode 100644 index 000000000..5bf13711e --- /dev/null +++ b/docs/versioned-plugins/outputs/tcp-v6.2.1.asciidoc @@ -0,0 +1,239 @@ +:plugin: tcp +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.2.1 +:release_date: 2024-06-05 +:changelog_url: https://github.com/logstash-plugins/logstash-output-tcp/blob/v6.2.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Write events over a TCP socket. + +By default this plugin uses the `json` codec. In order to have each event json separated by a newline, use the `json_lines` codec. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-reconnect_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"client"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +When connect failed,retry interval in sec. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cacert"] +===== `ssl_cacert` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The SSL CA certificate, chainfile or CA path. The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate path + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `none` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key path + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/tcp-v6.2.2.asciidoc b/docs/versioned-plugins/outputs/tcp-v6.2.2.asciidoc new file mode 100644 index 000000000..4f301fea6 --- /dev/null +++ b/docs/versioned-plugins/outputs/tcp-v6.2.2.asciidoc @@ -0,0 +1,239 @@ +:plugin: tcp +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v6.2.2 +:release_date: 2025-04-29 +:changelog_url: https://github.com/logstash-plugins/logstash-output-tcp/blob/v6.2.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Write events over a TCP socket. + +By default this plugin uses the `json` codec. In order to have each event json separated by a newline, use the `json_lines` codec. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-reconnect_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cacert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_cert>> |a valid filesystem path|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enable>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|__Deprecated__ +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verify>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"client"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +When connect failed,retry interval in sec. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cacert"] +===== `ssl_cacert` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The SSL CA certificate, chainfile or CA path. The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cert"] +===== `ssl_cert` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL certificate path + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `none` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enable"] +===== `ssl_enable` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_enabled>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key path + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verify"] +===== `ssl_verify` +deprecated[6.2.0, Replaced by <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>] + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Verify the identity of the other end of the SSL connection against the CA. +For input, sets the field `sslsubject` to that of the client certificate. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/tcp-v7.0.0.asciidoc b/docs/versioned-plugins/outputs/tcp-v7.0.0.asciidoc new file mode 100644 index 000000000..936a64770 --- /dev/null +++ b/docs/versioned-plugins/outputs/tcp-v7.0.0.asciidoc @@ -0,0 +1,218 @@ +:plugin: tcp +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.0 +:release_date: 2025-01-10 +:changelog_url: https://github.com/logstash-plugins/logstash-output-tcp/blob/v7.0.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Write events over a TCP socket. + +By default this plugin uses the `json` codec. In order to have each event json separated by a newline, use the `json_lines` codec. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-reconnect_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"client"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +When connect failed,retry interval in sec. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `none` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key path + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== TCP Output Obsolete Configuration Options + +WARNING: As of version `6.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ssl_cacert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| ssl_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| ssl_enable |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> in `server` mode and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> in `client` mode +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/tcp-v7.0.1.asciidoc b/docs/versioned-plugins/outputs/tcp-v7.0.1.asciidoc new file mode 100644 index 000000000..af3c0651f --- /dev/null +++ b/docs/versioned-plugins/outputs/tcp-v7.0.1.asciidoc @@ -0,0 +1,218 @@ +:plugin: tcp +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v7.0.1 +:release_date: 2025-04-29 +:changelog_url: https://github.com/logstash-plugins/logstash-output-tcp/blob/v7.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Tcp output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Write events over a TCP socket. + +By default this plugin uses the `json` codec. In order to have each event json separated by a newline, use the `json_lines` codec. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Tcp Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +NOTE: As of version `7.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<<{version}-plugins-{type}s-{plugin}-obsolete-options>> for more details. + + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["server", "client"]`|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|Yes +| <<{version}-plugins-{type}s-{plugin}-reconnect_interval>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_cipher_suites>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["none", "optional", "required"]`|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_supported_protocols>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "none"]`|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +When mode is `server`, the address to listen on. +When mode is `client`, the address to connect to. + +[id="{version}-plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"client"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +When mode is `server`, the port to listen on. +When mode is `client`, the port to connect to. + +[id="{version}-plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `10` + +When connect failed,retry interval in sec. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Validate client certificate or certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="{version}-plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="{version}-plugins-{type}s-{plugin}-ssl_client_authentication"] +===== `ssl_client_authentication` + + * Value can be any of: `none`, `optional`, `required` + * Default value is `none` + +Controls the server's behavior in regard to requesting a certificate from client connections: +`none` disables the client authentication. `required` forces a client to present a certificate, while `optional` requests a client certificate +but the client is not required to present one. + +When mutual TLS is enabled (`optional` or `required`), the certificate presented by the client must be signed by trusted +<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> (CAs). +Please note that the server does not validate the client certificate CN (Common Name) or SAN (Subject Alternative Name). + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `server` and <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set. + + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL key path + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * Default value is `nil` + +SSL key passphrase + +[id="{version}-plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +NOTE: This setting can be used only if <<{version}-plugins-{type}s-{plugin}-mode>> is `client`. + +[id="{version}-plugins-{type}s-{plugin}-obsolete-options"] +==== TCP Output Obsolete Configuration Options + +WARNING: As of version `6.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. + + +[cols="<,<",options="header",] +|======================================================================= +|Setting|Replaced by +| ssl_cacert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> +| ssl_cert |<<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +| ssl_enable |<<{version}-plugins-{type}s-{plugin}-ssl_enabled>> +| ssl_verify |<<{version}-plugins-{type}s-{plugin}-ssl_client_authentication>> in `server` mode and <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> in `client` mode +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/docs/versioned-plugins/outputs/websocket-index.asciidoc b/docs/versioned-plugins/outputs/websocket-index.asciidoc index 9de386d50..5a094e5d8 100644 --- a/docs/versioned-plugins/outputs/websocket-index.asciidoc +++ b/docs/versioned-plugins/outputs/websocket-index.asciidoc @@ -5,12 +5,14 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2024-01-11 | <> | 2018-04-06 | <> | 2017-11-13 | <> | 2017-08-16 | <> | 2017-06-23 |======================================================================= +include::websocket-v3.1.0.asciidoc[] include::websocket-v3.0.5.asciidoc[] include::websocket-v3.0.4.asciidoc[] include::websocket-v3.0.3.asciidoc[] diff --git a/docs/versioned-plugins/outputs/websocket-v3.1.0.asciidoc b/docs/versioned-plugins/outputs/websocket-v3.1.0.asciidoc new file mode 100644 index 000000000..55d536e0d --- /dev/null +++ b/docs/versioned-plugins/outputs/websocket-v3.1.0.asciidoc @@ -0,0 +1,69 @@ +:plugin: websocket +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.1.0 +:release_date: 2024-01-11 +:changelog_url: https://github.com/logstash-plugins/logstash-output-websocket/blob/v3.1.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Websocket output plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output runs a websocket server and publishes any +messages to all connected websocket clients. + +You can connect to it with ws://:/ + +If no clients are connected, any messages received are ignored. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Websocket Output Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-host>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-port>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +output plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"0.0.0.0"` + +The address to serve websocket data from + +[id="{version}-plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3232` + +The port to serve websocket data from + + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/plugin_docs_last_generated_time.txt b/plugin_docs_last_generated_time.txt new file mode 100644 index 000000000..d15e2db6c --- /dev/null +++ b/plugin_docs_last_generated_time.txt @@ -0,0 +1 @@ +2023-11-30 00:00:00 UTC