diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index 151a87b4d6..64f184d921 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -76,6 +76,9 @@ OZONE_RUNNER_VERSION=20230615-1 # Trino Configuration TRINO_VERSION=latest +# Open Search +OPENSEARCH_VERSION=1.3.19 + # Debug Configuration DEBUG_ADMIN=false DEBUG_USERSYNC=false diff --git a/dev-support/ranger-docker/Dockerfile.ranger-opensearch b/dev-support/ranger-docker/Dockerfile.ranger-opensearch new file mode 100644 index 0000000000..92c1c5a41f --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-opensearch @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} + +ARG OPENSEARCH_VERSION + +VOLUME /etc/keytabs + +COPY ./dist/version /home/ranger/dist/ +COPY ./downloads/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz /home/ranger/dist/ + +COPY ./scripts/opensearch/ranger-opensearch-setup.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/opensearch/ranger-opensearch.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/opensearch/opensearch.yml ${RANGER_SCRIPTS}/ +COPY ./scripts/opensearch/opensearch-jaas.conf ${RANGER_SCRIPTS}/ +COPY ./scripts/wait_for_keytab.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/wait_for_testusers_keytab.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/kdc/krb5.conf /etc/krb5.conf + +# Create opensearch user and group +RUN groupadd -g 3002 opensearch && \ + useradd -u 3002 -g opensearch -G hadoop -s /bin/bash opensearch + +# Extract and setup OpenSearch +RUN tar xvfz /home/ranger/dist/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz --directory=/opt/ && \ + ln -s /opt/opensearch-${OPENSEARCH_VERSION} /opt/opensearch && \ + rm -f /home/ranger/dist/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz && \ + mkdir -p /opt/opensearch/data /opt/opensearch/logs && \ + chown -R opensearch:hadoop /opt/opensearch* && \ + chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh && \ + chmod 755 ${RANGER_SCRIPTS}/wait_for_testusers_keytab.sh && \ + chmod 755 ${RANGER_SCRIPTS}/ranger-opensearch-setup.sh && \ + chmod 755 ${RANGER_SCRIPTS}/ranger-opensearch.sh + +ENV OPENSEARCH_HOME=/opt/opensearch +ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/opensearch/bin + +ENTRYPOINT [ "/home/ranger/scripts/ranger-opensearch.sh" ] + diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md index 838689b60b..39a89440f1 100644 --- a/dev-support/ranger-docker/README.md +++ b/dev-support/ranger-docker/README.md @@ -30,11 +30,11 @@ Use Dockerfiles in this directory to create docker images and run them to build - Set ```dev-support/ranger-docker``` as your working directory. -- Execute following command to download necessary archives to setup Ranger/HDFS/Hive/HBase/Kafka/Knox/Ozone services: +- Execute following command to download necessary archives to setup Ranger/HDFS/Hive/HBase/Kafka/Knox/Ozone/OpenSearch services: ~~~ chmod +x download-archives.sh # use a subset of the below to download specific services - ./download-archives.sh hadoop hive hbase kafka knox ozone + ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch ~~~ - Execute following commands to set environment variables to build Apache Ranger docker containers: @@ -102,6 +102,10 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-ozone.yml u ~~~ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-trino.yml up -d ~~~ +#### Bring up opensearch container: +~~~ +docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-opensearch.yml up -d +~~~ Similarly, check the `depends` section of the `docker-compose.ranger-service.yaml` file and add docker-compose files for these services when trying to bring up the `service` container. #### Bring up all containers diff --git a/dev-support/ranger-docker/docker-compose.ranger-opensearch.yml b/dev-support/ranger-docker/docker-compose.ranger-opensearch.yml new file mode 100644 index 0000000000..fa29c3ff40 --- /dev/null +++ b/dev-support/ranger-docker/docker-compose.ranger-opensearch.yml @@ -0,0 +1,42 @@ +services: + ranger-opensearch: + build: + context: . + dockerfile: Dockerfile.ranger-opensearch + args: + - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} + - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} + - OPENSEARCH_VERSION=${OPENSEARCH_VERSION} + - KERBEROS_ENABLED=${KERBEROS_ENABLED} + image: ranger-opensearch + container_name: ranger-opensearch + hostname: ranger-opensearch.rangernw + volumes: + - ./dist/keytabs/ranger-opensearch:/etc/keytabs + - opensearch-data:/opt/opensearch/data + - opensearch-logs:/opt/opensearch/logs + stdin_open: true + tty: true + networks: + - ranger + ports: + - "9200:9200" + - "9300:9300" + depends_on: + ranger-kdc: + condition: service_started + ranger: + condition: service_started + environment: + - OPENSEARCH_VERSION=${OPENSEARCH_VERSION} + - KERBEROS_ENABLED=${KERBEROS_ENABLED} + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + +volumes: + opensearch-data: + opensearch-logs: + +networks: + ranger: + name: rangernw + diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh index 1ce4240e4d..329b823a52 100755 --- a/dev-support/ranger-docker/download-archives.sh +++ b/dev-support/ranger-docker/download-archives.sh @@ -87,6 +87,9 @@ else then tar xvfz downloads/ozone-${OZONE_VERSION}.tar.gz --directory=downloads/ fi + elif [[ $arg == 'opensearch' ]] + then + downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION} else echo "Passed argument $arg is invalid!" fi diff --git a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh index 4a4fcf4602..d3bcc78e12 100644 --- a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh +++ b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh @@ -100,6 +100,9 @@ function create_keytabs() { create_principal_and_keytab knox ranger-knox create_principal_and_keytab HTTP ranger-solr + + create_principal_and_keytab opensearch ranger-opensearch + create_principal_and_keytab HTTP ranger-opensearch } function create_testusers() { @@ -126,7 +129,7 @@ if [ ! -f $DB_DIR/principal ]; then echo "Database initialized" create_keytabs - create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino + create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino ranger-opensearch else echo "KDC DB already exists; skipping create" fi diff --git a/dev-support/ranger-docker/scripts/opensearch/opensearch-jaas.conf b/dev-support/ranger-docker/scripts/opensearch/opensearch-jaas.conf new file mode 100644 index 0000000000..a8836a2258 --- /dev/null +++ b/dev-support/ranger-docker/scripts/opensearch/opensearch-jaas.conf @@ -0,0 +1,18 @@ +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="/etc/keytabs/opensearch.keytab" + principal="opensearch/ranger-opensearch.rangernw@EXAMPLE.COM"; +}; + +Server { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="/etc/keytabs/HTTP.keytab" + principal="HTTP/ranger-opensearch.rangernw@EXAMPLE.COM"; +}; + diff --git a/dev-support/ranger-docker/scripts/opensearch/opensearch.yml b/dev-support/ranger-docker/scripts/opensearch/opensearch.yml new file mode 100644 index 0000000000..d63405043b --- /dev/null +++ b/dev-support/ranger-docker/scripts/opensearch/opensearch.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OpenSearch Configuration +cluster.name: ranger-opensearch-cluster +node.name: ranger-opensearch.rangernw + +# Network settings +network.host: ranger-opensearch.rangernw +http.port: 9200 +transport.port: 9300 + +# Discovery settings +discovery.type: single-node + +# Path settings +path.data: /opt/opensearch/data +path.logs: /opt/opensearch/logs + +# Memory settings +bootstrap.memory_lock: false + +# Disable OpenSearch Security Plugin. +# Ranger OpenSearch plugin integration is not configured in this image. +# This can be enabled when Ranger Plugin is configured. +plugins.security.disabled: true + +# Allow HTTP methods +http.cors.enabled: true +http.cors.allow-origin: "http://localhost" +http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE +http.cors.allow-headers: "X-Requested-With, Content-Type, Content-Length, Authorization" + +# Kerberos authentication is enabled via JAAS configuration +# See opensearch-jaas.conf for Kerberos principal and keytab settings +# JVM is configured with: -Djava.security.auth.login.config and -Djava.security.krb5.conf + diff --git a/dev-support/ranger-docker/scripts/opensearch/ranger-opensearch-setup.sh b/dev-support/ranger-docker/scripts/opensearch/ranger-opensearch-setup.sh new file mode 100755 index 0000000000..b9dcdedf56 --- /dev/null +++ b/dev-support/ranger-docker/scripts/opensearch/ranger-opensearch-setup.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wait for Kerberos keytabs (enabled by default) +if [ "${KERBEROS_ENABLED}" != "false" ] +then + echo "Kerberos is enabled, waiting for keytabs..." + ${RANGER_SCRIPTS}/wait_for_keytab.sh opensearch.keytab + ${RANGER_SCRIPTS}/wait_for_keytab.sh HTTP.keytab + ${RANGER_SCRIPTS}/wait_for_testusers_keytab.sh +else + echo "Kerberos is disabled" +fi + +# Copy configuration files +cp ${RANGER_SCRIPTS}/opensearch.yml ${OPENSEARCH_HOME}/config/ +cp ${RANGER_SCRIPTS}/opensearch-jaas.conf ${OPENSEARCH_HOME}/config/ + +# Set ownership +chown -R opensearch:hadoop ${OPENSEARCH_HOME}/ + +echo "OpenSearch setup completed successfully" + diff --git a/dev-support/ranger-docker/scripts/opensearch/ranger-opensearch.sh b/dev-support/ranger-docker/scripts/opensearch/ranger-opensearch.sh new file mode 100755 index 0000000000..a9f27aa5e3 --- /dev/null +++ b/dev-support/ranger-docker/scripts/opensearch/ranger-opensearch.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! -e ${OPENSEARCH_HOME}/.setupDone ] +then + if "${RANGER_SCRIPTS}"/ranger-opensearch-setup.sh; + then + touch "${OPENSEARCH_HOME}"/.setupDone + else + echo "OpenSearch Setup Script didn't complete proper execution." >&2 + exit 1 + fi +fi + +# Start OpenSearch as opensearch user with Kerberos enabled by default +if [ "${KERBEROS_ENABLED}" != "false" ]; then + echo "Starting OpenSearch with Kerberos authentication enabled..." + su -c "cd ${OPENSEARCH_HOME} && OPENSEARCH_JAVA_OPTS=\"${OPENSEARCH_JAVA_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/opensearch/config/opensearch-jaas.conf\" ./bin/opensearch" opensearch +else + echo "Starting OpenSearch without Kerberos..." + su -c "cd ${OPENSEARCH_HOME} && ./bin/opensearch" opensearch +fi +