diff --git a/.gitignore b/.gitignore index b440d2b597..35096783dd 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ linklint-*.zip linklint/ .checkstyle **/.checkstyle + +# detritus produced by kuttl +kubeconfig* diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile index 5b7aac29a9..17bcf30645 100644 --- a/dev-support/jenkins/Dockerfile +++ b/dev-support/jenkins/Dockerfile @@ -20,16 +20,29 @@ FROM hadolint/hadolint:latest-debian as hadolint FROM maven:3.8-jdk-8 - -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ - binutils \ - git \ - rsync \ - shellcheck \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=SC1091 +RUN install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && chmod a+r /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get -q update \ + && apt-get -q install --no-install-recommends -y \ + binutils=* \ + ca-certificates=* \ + curl=* \ + docker-buildx-plugin=* \ + docker-ce-cli=* \ + git=* \ + gnupg=* \ + rsync=* \ + shellcheck=* \ + wget=* \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY --from=hadolint /bin/hadolint /bin/hadolint diff --git a/dev-support/jenkins/jenkins_precommit_github_yetus.sh b/dev-support/jenkins/jenkins_precommit_github_yetus.sh index 2cede2f047..40f3ead1a9 100755 --- a/dev-support/jenkins/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins/jenkins_precommit_github_yetus.sh @@ -111,6 +111,8 @@ YETUS_ARGS+=("--tests-filter=test4tests") # Dockerfile since we don't want to use the auto-pulled version. YETUS_ARGS+=("--docker") YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") +# enabled docker-in-docker so that we can build container images +YETUS_ARGS+=("--dockerind=true") YETUS_ARGS+=("--mvn-custom-repos") YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}") # effectively treat dev-support as a custom maven module @@ -127,4 +129,4 @@ YETUS_ARGS+=("--proclimit=5000") echo "Launching yetus with command line:" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}" -/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" \ No newline at end of file +/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" diff --git a/hbase-kubernetes-deployment/README.md b/hbase-kubernetes-deployment/README.md new file mode 100644 index 0000000000..d240eacaae --- /dev/null +++ b/hbase-kubernetes-deployment/README.md @@ -0,0 +1,27 @@ + + +# HBase Kubernetes Deployment + +This module contains Kubernetes configurations suitable for deploying ZooKeeper, HDFS, and HBase. +It supports several physical topologies, including: + * a minimum deployment footprint consisting of a single instance of each architectural component. + * a high-avaiability deployment footprint consisting of redundancies for each architectural + component. + * a deployment where HBase region servers and HDFS data nodes share a pod, enabling short-circuit + read between them. diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/README.md new file mode 100644 index 0000000000..3a1b093832 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/README.md @@ -0,0 +1,56 @@ + + +# hbase-kubernetes-hadoop-image + +There is a contract that must be maintained between the container image run in the pod and the +infrastructure that launched the pod. Details like paths, users/groups, permissions, and +environment variables must align so that the deployment layer can pass runtime concerns down to +the container. + +Start with the official hadoop image and extend it from there. Note that `apache/hadoop:3` is only +published for `linux/amd64` at this time. + +## Build + +Input arguments are managed by [docker-bake.override.hcl](./docker-bake.override.hcl). + +Start by creating a buildx context that supports at least `linux/amd64` images. If you've created +this context previously, it's enough to ensure that it's active via `docker buildx ls`. + +```shell +$ docker buildx create \ + --driver docker-container \ + --platform linux/amd64,linux/arm64 \ + --use \ + --bootstrap +``` + +Finally, build the image using `mvn package`, or manually, using, + +```shell +$ docker buildx bake \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl \ + --set '*.platform=linux/amd64' \ + --pull \ + --load +``` + +This exports an image to your local repository that is tagged as +`${USER}/hbase/operator-tools/hadoop:latest`. diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/pom.xml b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/pom.xml new file mode 100644 index 0000000000..b4890a519e --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/pom.xml @@ -0,0 +1,104 @@ + + + + 4.0.0 + + hbase-kubernetes-deployment + org.apache.hbase.operator.tools + ${revision} + .. + + + hbase-kubernetes-hadoop-image + Apache HBase - Kubernetes Hadoop Image + A container image to for running Hadoop in Kubernetes. + pom + + + + linux/amd64 + + + + + + + + org.codehaus.mojo + exec-maven-plugin + + true + + + + docker-buildx-bake-print + + exec + + package + + /usr/bin/env + + sh + -c + + 2>&1 \ + docker buildx bake \ + --print \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl + + + + + + docker-buildx-bake + + exec + + package + + /usr/bin/env + + sh + -c + + 2>&1 \ + docker buildx bake \ + --progress plain \ + --pull \ + --load \ + --set *.platform=${container_image.platforms} \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl + + + + + + + + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/Dockerfile b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/Dockerfile new file mode 100644 index 0000000000..8c514fa96d --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/Dockerfile @@ -0,0 +1,57 @@ +# syntax=docker/dockerfile:1.4 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# hadolint global ignore=DL3020 + +ARG BASE_IMG='apache/hadoop' +ARG BASE_TAG='3' +ARG BASE_IMG_ALPINE='alpine' +ARG BASE_IMG_TAG_ALPINE='latest' + +FROM ${BASE_IMG_ALPINE}:${BASE_IMG_TAG_ALPINE} as alpine + +## -- Stage installed_jmx_exporter -- +# A stage that prepares the JMX Exporter installation directory. The jar must be in well-known +# location so can refer to it in command yaml value. +FROM alpine AS installed_jmx_exporter +ARG JMX_PROMETHEUS_JAR_URL +ADD --link ${JMX_PROMETHEUS_JAR_URL} /tmp/jmx_prometheus_javaagent.jar + +FROM ${BASE_IMG}:${BASE_TAG} as final +ARG CORRETTO_KEY_URL +ARG CORRETTO_KEY +ARG CORRETTO_REPO_URL +ARG CORRETTO_REPO +USER root +ADD --link ${CORRETTO_KEY_URL} /tmp/ +ADD --link ${CORRETTO_REPO_URL} /etc/yum.repos.d/ + +RUN rpm --import "/tmp/${CORRETTO_KEY}" \ + && yum -y remove java-1.8.0-* \ + && yum -y update \ + && yum -y install java-17-amazon-corretto-devel \ + && yum -y clean all \ + && rm -rf /var/cache + +COPY --from=installed_jmx_exporter \ + --chown=hadoop:users \ + /tmp/jmx_prometheus_javaagent.jar /opt/hadoop/jmx_prometheus_javaagent.jar + +ENV JMX_PROMETHEUS_JAR /opt/hadoop/jmx_prometheus_javaagent.jar +ENV USER='hadoop' +USER ${USER} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/docker-bake.hcl b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/docker-bake.hcl new file mode 100644 index 0000000000..0040cfad29 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/docker-bake.hcl @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A convenience script for build the kuttl image. +# See hbase-kubernetes-deployment/dockerfiles/kuttl/README.md +# + +variable BASE_IMG { + default = "apache/hadoop" +} +variable BASE_TAG { + default = "3" +} +variable USER { + default = "apache" +} +variable IMAGE_TAG { + default = "latest" +} +variable IMAGE_NAME { + default = "${USER}/hbase/operator-tools/hadoop" +} +variable CORRETTO_KEY_URL {} +variable CORRETTO_KEY {} +variable CORRETTO_REPO_URL {} +variable CORRETTO_REPO {} +variable JMX_PROMETHEUS_JAR_URL {} +variable JMX_PROMETHEUS_JAR {} + +group default { + targets = [ "hadoop" ] +} + +target hadoop { + dockerfile = "src/main/docker/Dockerfile" + args = { + BASE_IMG = BASE_IMG + BASE_TAG = BASE_TAG + CORRETTO_KEY_URL = CORRETTO_KEY_URL + CORRETTO_KEY = CORRETTO_KEY + CORRETTO_REPO_URL = CORRETTO_REPO_URL + CORRETTO_REPO = CORRETTO_REPO + JMX_PROMETHEUS_JAR_URL = JMX_PROMETHEUS_JAR_URL + JMX_PROMETHEUS_JAR = JMX_PROMETHEUS_JAR + } + target = "final" + platforms = [ + # upstream image only provides linux/amd64 + "linux/amd64" + ] + tags = [ "${IMAGE_NAME}:${IMAGE_TAG}" ] +} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/docker-bake.override.hcl b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/docker-bake.override.hcl new file mode 100644 index 0000000000..cbe27ddc65 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/docker-bake.override.hcl @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Externalize default values of build parameters and document how to retrieve them. +# + +function "basename" { + params = [a] + result = split("/", a)[length(split("/", a)) - 1] +} + +variable CORRETTO_KEY_URL { + default = "https://yum.corretto.aws/corretto.key" +} + +variable CORRETTO_KEY { + default = "${basename(CORRETTO_KEY_URL)}" +} + +variable CORRETTO_REPO_URL { + default = "https://yum.corretto.aws/corretto.repo" +} + +variable CORRETTO_REPO { + default = "${basename(CORRETTO_REPO_URL)}" +} + +variable JMX_PROMETHEUS_JAR_URL { + default = "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar" +} + +variable JMX_PROMETHEUS_JAR { + default = "${basename(JMX_PROMETHEUS_JAR_URL)}" +} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml new file mode 100644 index 0000000000..c33c53cf00 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml @@ -0,0 +1,100 @@ + + + + 4.0.0 + + hbase-kubernetes-deployment + org.apache.hbase.operator.tools + ${revision} + .. + + + hbase-kubernetes-kustomize + Apache HBase - Kubernetes Kustomize + Kustomize deployment descriptors. + pom + + + + org.apache.hbase.operator.tools + hbase-kubernetes-testing-image + pom + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + + true + ${skipTests} + + + + kuttl-unit-tests + test + + exec + + + /usr/bin/env + + sh + -c + + 2>&1 \ + src/test/resources/mvn_exec_run_kuttl.sh \ + --config src/test/resources/kuttl-test-unit.yaml \ + --artifacts-dir target/kuttl-reports + + + + + + kuttl-integration-tests + verify + + exec + + + /usr/bin/env + + sh + -c + + 2>&1 \ + src/test/resources/mvn_exec_run_kuttl.sh \ + --config src/test/resources/kuttl-test-integration.yaml \ + --artifacts-dir target/kuttl-reports + + + + + + + + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/README.md new file mode 100644 index 0000000000..da3c6cd211 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/README.md @@ -0,0 +1,39 @@ + + +# Base + +Some values such as SERVICE name, SERVICEACCOUNT name, +and RBAC role are hard-coded in the environment-configmap.yaml +and supplied into the pods as environment variables. Other +hardcodings include the service name ('hadoop') and the +namespace we run in (also 'hadoop'). + +The hadoop Configuration system can interpolate environment variables +into '\*.xml' file values ONLY. See +[Configuration Javadoc](http://hadoop.apache.org/docs/current/api/org/apache/hadoop/conf/Configuration.html) + +...but we can not do interpolation of SERVICE name into '\*.xml' file key names +as is needed when doing HA in hdfs-site.xml... so for now, we have +hard-codings in 'hdfs-site.xml' key names. For example, the property key name +`dfs.ha.namenodes.hadoop` has the SERVICE name ('hadoop') in it or the key +`dfs.namenode.http-address.hadoop` (TODO: Fix/Workaround). + +Edit of pod resources or jvm args for a process are +done in place in the yaml files or in kustomization +replacements in overlays. diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/delete-format-hdfs-configmap-job.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/delete-format-hdfs-configmap-job.yaml new file mode 100644 index 0000000000..cc52f4d36e --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/delete-format-hdfs-configmap-job.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Job to delete the 'format-hdfs' configmap after hdfs has come up +# successfully. The 'format-hdfs' configmap is added by running +# 'kubectl -n hadoop apply -k tools/format-hdfs' (You need the +# '-n hadoop' to apply the configmap to the 'hadoop' namespace). +# Add the configmap if you want hdfs to format the filesystem. +# Do this on initial install only or if you want to clean out +# the current HDFS data. +# +# If the 'format-hdfs' configmap is NOT present, this Job exits/completes. +# Otherwise, it keeps probing until HDFS is up and healthy, and then +# this job removes the 'format-hdfs' configmap. The presence of the +# 'format-hdfs' configmap is checked by all hdfs pods on startup. If +# the configmap is present, they clean out their data directories and run +# format/recreate of their data directories. To install the 'format-hdfs' +# configmap, do it before launch of hdfs. See tools/format-hdfs. +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: delete-format-hdfs-configmap +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + containers: + - image: hadoop + name: delete-format-hdfs-configmap + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + - |- + set -xe + # See if 'format-hdfs' configmap is present. + # If not, then there is nothing for this job to do, complete, exit 0. + /tmp/scripts/exists_configmap.sh format-hdfs || { + echo "No 'format-hdfs' configmap found so no work to do; exiting" + exit 0 + } + # The `format-hdfs`` configmap is present. Remove it after HDFS is fully up. + /tmp/scripts/jmxping.sh namenode ${HADOOP_SERVICE} + /tmp/scripts/jmxping.sh datanode ${HADOOP_SERVICE} + # TODO: Should we check if ha and if so, if a NN active... get a report on health? + # HDFS is up. Delete the format-hdfs flag. + /tmp/scripts/delete_configmap.sh format-hdfs + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + volumeMounts: + - mountPath: /tmp/scripts + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - mountPath: /tmp/scratch + name: scratch + serviceAccountName: hadoop + restartPolicy: Never + volumes: + - configMap: + name: scripts + defaultMode: 0555 + name: scripts + # Scratch dir is location where init containers place items for later use + # by the main containers when they run. + - emptyDir: {} + name: scratch diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/environment-configmap.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/environment-configmap.yaml new file mode 100644 index 0000000000..d018c22c82 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/environment-configmap.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Common environment variables shared across pods. +# Include w/ the 'envFrom:' directive. +# We have to be pendantic in here. We cannot have a value +# refer to a define made earlier; the interpolation +# doesn't work. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: environment +data: + DOMAIN: svc.cluster.local + # HADOOP_HOME, HADOOP_HDFS_HOME, etc., and HBASE_HOME are provided by the images. + # + # The headless-service pods in our statefulsets come up in. + # See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id + # The headless-service is defined in the adjacent rbac.yaml. + # Matches the serviceName we have on our statefulsets. + # Required that we create it according to https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#limitations + HADOOP_SERVICE: hadoop + # dfs.http.policy + # If HTTPS_ONLY or HTTPS_OR_HTTP then we'll depend on https in UI and jmx'ing + # and will adjust schema and ports accordingly. If https, we need to get certificates + # so cert-manager, etc., needs to be instaled. + HTTP_POLICY: HTTP_ONLY + DFS_HTTPS_ENABLE: "false" + HBASE_SSL_ENABLED: "false" + HTTP_AUTH: kerberos + # The insecure port for now. + DATANODE_DATA_DIR: /data00/dn + JOURNALNODE_DATA_DIR: /data00/jn + NAMENODE_DATA_DIR: /data00/nn + HDFS_AUDIT_LOGGER: INFO,RFAAUDIT + HADOOP_DAEMON_ROOT_LOGGER: INFO,RFA,CONSOLE + HADOOP_ROOT_LOGGER: INFO,RFA,CONSOLE + HADOOP_SECURITY_LOGGER: INFO,RFAS + HADOOP_CONF_DIR: /etc/hadoop + HADOOP_LOG_DIR: /var/log/hadoop + HADOOP_SECURE_LOG: /var/log/hadoop + HBASE_ROOT_LOGGER: DEBUG,RFA,console + HBASE_LOG_DIR: /var/log/hbase + HBASE_CONF_DIR: /etc/hbase + # if [ "$HBASE_NO_REDIRECT_LOG" != "" ]; then ... so we are asking for NO redirect of logs. + HBASE_NO_REDIRECT_LOG: "true" + HBASE_MANAGES_ZK: "false" + DFS_REPLICATION: "1" + # What percentage of the container memory to give over to the JVM. + # Be aware that we look at the container resource limit, NOT request: e.g. if + # the resource request memory is set to 8G and the limit is 16G and the + # JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT is 50 as in 50%, + # the heap will be set to 8G: i.e. 1/2 of the 16G limit. + # ip-172-18-132-227.us-west-2.compute.internal + # See https://dzone.com/articles/best-practices-java-memory-arguments-for-container + JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT: "45" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/java.security b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/java.security new file mode 100644 index 0000000000..c5c4f0403d --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/java.security @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networkaddress.cache.ttl=1 +networkaddress.cache.negative.ttl=0 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/jmxexporter.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/jmxexporter.yaml new file mode 100644 index 0000000000..4dd20fa0e4 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/jmxexporter.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# We run the jmxexporter on most all processes to convert jmx metrics to prometheus. +# This is the config file it uses. +# +# Don't lowercase. Leave the metrics in camelcase. Do this because while +# jmxexport can lowercase metrics names, telegraf can't. +# +#lowercaseOutputName: false +#lowercaseOutputLabelNames: false +# From https://godatadriven.com/blog/monitoring-hbase-with-prometheus/ +#rules: +# - pattern: HadoopNamespace_([^\W_]+)_table_([^\W_]+)_region_([^\W_]+)_metric_(\w+) +# name: HBase_metric_$4 +# labels: +# namespace: "$1" +# table: "$2" +# region: "$3" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/kustomization.yaml new file mode 100644 index 0000000000..43dd57c930 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/kustomization.yaml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configMapGenerator: +- name: hadoop-configuration + # Base set of hadoop configurations. Overlays will add to the set here. + files: + - log4j.properties=log4j.properties.hadoop +- name: scripts + # Useful scripts + files: + - scripts/jmxping.sh + - scripts/apiserver_access.sh + - scripts/get_statefulset_replica_count.sh + - scripts/get_statefulset.sh + - scripts/exists_configmap.sh + - scripts/delete_configmap.sh + - scripts/topology.sh + - scripts/describe_node.sh + - scripts/get_node_name_from_pod_IP.sh + - scripts/get_node_labels.sh + - scripts/get_node_labels_from_pod_IP.sh + - scripts/log.sh + options: + disableNameSuffixHash: true +- name: global-files + # Add files used by most/all processes into a global configuration configmap + # accessible to all processes. The environment-configmap defines env varibles used by + # all processes and pods. This configmap loads files used by each process. + files: + - jmxexporter.yaml + - java.security + - ssl-client.xml + - ssl-server.xml + options: + disableNameSuffixHash: true + +secretGenerator: +- name: keystore-password + type: Opaque + options: + disableNameSuffixHash: true + literals: + - password=changeit + +resources: +- namespace.yaml +# Global environment variables read in by pods +- environment-configmap.yaml +- rbac.yaml +- delete-format-hdfs-configmap-job.yaml +# These depend on cert-manager being installed. +# See https://cert-manager.io/docs/installation/ +#- clusterissuer.yaml +#- certificate.yaml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/log4j.properties.hadoop b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/log4j.properties.hadoop new file mode 100644 index 0000000000..df7cf7b6b8 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/log4j.properties.hadoop @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hadoop.console.threshold=LOG +hadoop.log.maxbackupindex=20 +hadoop.log.maxfilesize=256MB +hadoop.root.logger=TRACE,CONSOLE +hadoop.security.log.file=SecurityAuth-${user.name}.audit +hadoop.security.log.maxbackupindex=20 +hadoop.security.log.maxfilesize=256MB +hadoop.security.logger=INFO,RFAS +hdfs.audit.log.maxbackupindex=20 +hdfs.audit.log.maxfilesize=256MB +hdfs.audit.logger=INFO,RFAAUDIT +log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n +log4j.appender.CONSOLE.Threshold=${hadoop.console.threshold} +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n +log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} +log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} +log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender +log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log +log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n +log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} +log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n +log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} +log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} +log4j.category.SecurityLogger=${hadoop.security.logger} +log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} +log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy=DEBUG +log4j.logger.org.apache.hadoop.net.NetworkTopology=DEBUG +log4j.rootLogger=${hadoop.root.logger} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/namespace.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/namespace.yaml new file mode 100644 index 0000000000..f3e73a6eea --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/namespace.yaml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Allow the namespace of the user's Kustomization to be the destination of the deployment. +# How to manage the namespace with Kustomize -- https://stackoverflow.com/a/71150557 +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ~~illegal_value_to_be_overridden_in_Kustomization~~ diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/rbac.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/rbac.yaml new file mode 100644 index 0000000000..29e9c89e5a --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/rbac.yaml @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Service and ServiceAccount names are hard-coded as 'hadoop'. +# RBAC Role name is also hard-coded as 'hadoop-role'. Service selects on +# an app named 'hadoop', another hard-coding. +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hadoop-role +rules: +- resources: + - configmaps + verbs: + - get + - delete + - list + apiGroups: + - '' +- resources: + - namespaces + verbs: + - get + - list + apiGroups: + - '' +- resources: + - statefulsets + verbs: + - get + - list + - patch + - update + apiGroups: + - 'apps' + - 'api' +- resources: + - pods + verbs: + - get + - list + - delete + - watch + apiGroups: + - '' +- resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + apiGroups: + - coordination.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hadoop-role-binding +subjects: +- kind: ServiceAccount + name: hadoop +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hadoop-role +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hadoop +--- +# Headless-service to cluster all our pods under +# Matches the ServiceAccount above referenced by statefulsets +# in their serviceName. +# See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id +# This is required for statefulsets. See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#limitations +apiVersion: v1 +kind: Service +metadata: + name: hadoop +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app: hadoop diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/apiserver_access.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/apiserver_access.sh new file mode 100755 index 0000000000..4a2929f2e8 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/apiserver_access.sh @@ -0,0 +1,26 @@ +#! /usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Defines used accessing the apiserver. +NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) +export NAMESPACE +APISERVER=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT +export APISERVER +CACERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt +export CACERT +TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) +export TOKEN diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/delete_configmap.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/delete_configmap.sh new file mode 100755 index 0000000000..58a3107580 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/delete_configmap.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Get the description of the named statefulset +set -x +configmap_name="${1}" +outfile="$(mktemp "/tmp/$(basename "$0").XXXX")" +trap '{ rm -f -- "$outfile"; }' EXIT +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# shellcheck source=/dev/null +source "${script_dir}/apiserver_access.sh" +# Following model described here: https://chengdol.github.io/2019/11/06/k8s-api/ +# http_code is the return status code +# From https://docs.okd.io/3.7/rest_api/api/v1.ConfigMap.html#Delete-api-v1-namespaces-namespace-configmaps-name +http_code=$(curl -w "%{http_code}" -sS -X DELETE --cacert "$CACERT" -H "Content-Type: application/json" -H "Accept: application/json, */*" -H "Authorization: Bearer $TOKEN" "$APISERVER/api/v1/namespaces/$NAMESPACE/configmaps/$configmap_name" -o "$outfile") +if [[ $http_code -ne 200 ]]; then + echo "{\"Result\": \"Failure\", \"httpReturnCode\":$http_code}" | jq '.' + exit 1 +fi +cat "$outfile" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/describe_node.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/describe_node.sh new file mode 100644 index 0000000000..df1ffa1e03 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/describe_node.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Get the description of the named node +set -x +node="${1}" +outfile="$(mktemp "/tmp/$(basename "$0")".XXXX)" +trap '{ rm -f -- "$outfile"; }' EXIT +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# shellcheck source=/dev/null +source "${script_dir}/apiserver_access.sh" +# Following model described here: https://chengdol.github.io/2019/11/06/k8s-api/ +# http_code is the return status code +http_code=$(curl -w "%{http_code}" -sS --cacert "$CACERT" -H "Content-Type: application/json" -H "Accept: application/json, */*" -H "Authorization: Bearer $TOKEN" "$APISERVER/api/v1/nodes/$node" -o "$outfile") +if [[ $http_code -ne 200 ]]; then + echo "{\"Result\": \"Failure\", \"httpReturnCode\":$http_code}" | jq '.' + exit 1 +fi +cat "$outfile" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/exists_configmap.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/exists_configmap.sh new file mode 100755 index 0000000000..7ed9a8fb27 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/exists_configmap.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Check passed in configmap exists. +# Also checks if configmap with the POD_NAME exists too. +# Returns zero if found. +set -x +configmap_name="${1}" +outfile="$(mktemp "/tmp/$(basename "$0").XXXX")" +trap 'rm -f -- "$outfile"' EXIT +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# shellcheck source=/dev/null +source "${script_dir}/apiserver_access.sh" +# Following model described here: https://chengdol.github.io/2019/11/06/k8s-api/ +# http_code is the return status code +# From https://docs.okd.io/3.7/rest_api/api/v1.ConfigMap.html#Delete-api-v1-namespaces-namespace-configmaps-name +http_code=$(curl -w "%{http_code}" -sS --cacert "$CACERT" -H "Content-Type: application/json" -H "Accept: application/json, */*" -H "Authorization: Bearer $TOKEN" "$APISERVER/api/v1/namespaces/$NAMESPACE/configmaps/$configmap_name" -o "$outfile") +[[ $http_code -eq 200 ]] || ( + # The configmap does not exist. Look for a configmap with this POD_NAME as a suffix too. + http_code=$(curl -w "%{http_code}" -sS --cacert "$CACERT" -H "Content-Type: application/json" -H "Accept: application/json, */*" -H "Authorization: Bearer $TOKEN" "$APISERVER/api/v1/namespaces/$NAMESPACE/configmaps/$configmap_name.${POD_NAME}" -o "$outfile") + [[ $http_code -eq 200 ]] +) diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_labels.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_labels.sh new file mode 100644 index 0000000000..bb50a6560e --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_labels.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch the labels json object for named node +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +"${script_dir}/describe_node.sh" "${1}" | jq -r '.metadata.labels' diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_labels_from_pod_IP.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_labels_from_pod_IP.sh new file mode 100644 index 0000000000..7458de36b3 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_labels_from_pod_IP.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Get the labels json object of the node upon which the pod with the provided pod IP is running +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# shellcheck source=/dev/null +source "${script_dir}/log.sh" "$TOPOLOGY_LOG" # source log function; the $TOPOLOGY_LOG variable is set in topology.sh +nodeName=$("${script_dir}/get_node_name_from_pod_IP.sh" "${1}") # requesting node name based on pod IP +if [[ "$nodeName" == "null" ]] # if no node is found when querying with this pod IP +then + log -w "Unhandled case: Kubernetes instance not found for this pod IP" + echo "null" # null will get passed back to the topology caller; then when looking for the pertinent labels topology.sh will label this DN with the default rack +else + log "nodeName found in pod description: $nodeName" + nodeLabels="$("${script_dir}/get_node_labels.sh" "$nodeName")" # getting the labels of the Kube node the pod is running on + log "node metadata labels: $nodeLabels" + echo "$nodeLabels" +fi diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_name_from_pod_IP.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_name_from_pod_IP.sh new file mode 100644 index 0000000000..a0cb279671 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_node_name_from_pod_IP.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Get the name of the Kubernetes node with the provided hadoop pod IP +set -x +podIP="${1}" # this will be the IP of a datanode +outfile="$(mktemp "/tmp/$(basename "$0").XXXX")" +trap '{ rm -f -- "$outfile"; }' EXIT +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# shellcheck source=/dev/null +source "${script_dir}/apiserver_access.sh" +# Following model described here: https://chengdol.github.io/2019/11/06/k8s-api/ +# http_code is the return status code +http_code=$(curl -w "%{http_code}" -sS --cacert "$CACERT" -H "Content-Type: application/json" -H "Accept: application/json, */*" -H "Authorization: Bearer $TOKEN" "$APISERVER/api/v1/namespaces/hadoop/pods?fieldSelector=status.podIP%3D$podIP" -o "$outfile") +if [[ $http_code -ne 200 ]]; then + echo "{\"Result\": \"Failure\", \"httpReturnCode\":$http_code}" | jq '.' + exit 1 +fi + +# using jq, only return the name of the node containing this pod; jq will return null if no node is found +jq -r .items[0].spec.nodeName "$outfile" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_statefulset.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_statefulset.sh new file mode 100755 index 0000000000..b6a9162015 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_statefulset.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Get the description of the named statefulset +set -x +statefulset="${1}" +outfile="$(mktemp "/tmp/$(basename "$0").XXXX")" +trap '{ rm -f -- "$outfile"; }' EXIT +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# shellcheck source=/dev/null +source "${script_dir}/apiserver_access.sh" +# Following model described here: https://chengdol.github.io/2019/11/06/k8s-api/ +# http_code is the return status code +http_code=$(curl -w "%{http_code}" -sS --cacert "$CACERT" -H "Content-Type: application/json" -H "Accept: application/json, */*" -H "Authorization: Bearer $TOKEN" "$APISERVER/apis/apps/v1/namespaces/$NAMESPACE/statefulsets/$statefulset" -o "$outfile") +if [[ $http_code -ne 200 ]]; then + echo "{\"Result\": \"Failure\", \"httpReturnCode\":$http_code}" | jq '.' + exit 1 +fi +cat "$outfile" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_statefulset_replica_count.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_statefulset_replica_count.sh new file mode 100755 index 0000000000..ad1149a375 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/get_statefulset_replica_count.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch the replica count for named statefulset +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +"${script_dir}/get_statefulset.sh" "${1}" | jq '.spec.replicas' diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/jmxping.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/jmxping.sh new file mode 100755 index 0000000000..2cf7aabda1 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/jmxping.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Usage: jmxping.sh [] +# JMX ping that there are at least '' instances of '' +# running in the sub-domain specified by +# (See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id). +# If no '' supplied, we read the replica count from passed +# in '' statefulset from apiserver. +set -x +role="${1}" +service="${2}" +count_param="${3}" +# Schema +schema=http +if [[ ${HTTP_POLICY} == HTTPS_* ]]; then + schema=https +fi +# Jmxport to use +case "${role}" in + datanode) + jmxport=9864 + if [[ ${HTTP_POLICY} == HTTPS_* ]]; then + # If HTTP policy is https, use https jmx port. + jmxport=9865 + fi + ;; + namenode) + jmxport=9870 + if [[ ${HTTP_POLICY} == HTTPS_* ]]; then + # If HTTP policy is https, use https jmx port. + jmxport=9871 + fi + ;; + journalnode) + jmxport=8480 + if [[ ${HTTP_POLICY} == HTTPS_* ]]; then + # If HTTP policy is https, use https jmx port. + jmxport=8481 + fi + ;; + master) + jmxport=16010 + ;; + regionserver) + jmxport=16030 + ;; + *) + exit 1 + ;; +esac + +interval=5 +timeout=$((60 * 60)) +while ((timeout > 0)) +do + # The statefulset we depend on may not have deployed yet... so the first + # attempts at getting replicas may fail. + # https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash + replicas="$(/tmp/scripts/get_statefulset_replica_count.sh "$role")" + count=${count_param} + if [ "x" = "${count_param}x" ]; then + count=${replicas} + else + count=$((replicas < count_param? replicas : count_param )) + fi + seq_end=$(( count - 1 )) + total=0 + for i in $( seq 0 $seq_end ); do + # Url is http://journalnode-1:8480/jmx?qry=java.lang:type=OperatingSystem + url="${schema}://${role}-${i}.${service}:${jmxport}/jmx?qry=java.lang:type=OperatingSystem" + # Returns 1 if success, zero otherwise. + result=$(curl --cacert /tmp/scratch/ca.crt -v "$url" | grep -c SystemLoadAverage) + ((total+=result)) + ((total != count)) || exit 0 + done + timeout=$((timeout - interval)) + echo "Failed; sleeping $interval, then retrying for $timeout more seconds" + sleep $interval +done +echo "Timedout!" +exit 1 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/log.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/log.sh new file mode 100644 index 0000000000..f1b9b6a522 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/log.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# when sourcing log, first argument should be the file within $HADOOP_LOG_DIR that will be written to + +filename=${1} +LOG_FILEPATH="$HADOOP_LOG_DIR/$filename" + +# logs provided message to whichever filepath is provided when sourcing log.sh +# Use -e for error logging, -w for warning logs +# log [-ew] MESSAGE +log(){ + prefix="" # No prefix with default INFO-level logging + while getopts ":ew" arg; do + case $arg in + e) # change prefix to ERROR: in logs + prefix="ERROR:" + shift + ;; + w) # change prefix to WARNING: in logs + prefix="WARNING:" + shift + ;; + *) # what is this? + ;; + esac + done + message=${1} + echo "$(date +"%F %T") $prefix $message" >> "$LOG_FILEPATH" +} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/topology.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/topology.sh new file mode 100755 index 0000000000..d405136aff --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/scripts/topology.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Using topology script notion for HDFS rack awareness: https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/RackAwareness.html + +# This script takes in one or more datanode IPs as args and passes out rack name(s) for the pod(s) based on the EKS instance(s) they're running in. +# It will look for information about the EKS instance's partition placement group: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/placement-groups.html#placement-groups-partition +# As well as information about the EKS instance's availability zone according to AWS: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones + +# if partition placement group information is found (in the form of the $partition_group_label variable defined below), +# then the rack passed out will be "partition-group-". + +# Otherwise, the script will take in availability zone information, pass out a +# rack label like "availability-zone-". + +# Supposition here is that when datanodes crash, the namenodes will provide the same rack when the pod comes back up. +# This is the behavior that's been observed when terminating datanodes manually and watching topology logs as they re-initialize. + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +TOPOLOGY_LOG="topology.log" # filepath within $HADOOP_LOG_DIR wherein topology logs will be placed +export TOPOLOGY_LOG + +# shellcheck source=/dev/null +source "${script_dir}/log.sh" "$TOPOLOGY_LOG" +partition_group_label="partition_number" # this is an assumption made based on the Siri cluster at the moment; modify this variable if the Kube node label signifying placement groups is named differently + +log "argument(s) input to script: $*" +for dn_IP in "$@" +do + log "datanode IP: $dn_IP" + nodeLabels="$("${script_dir}/get_node_labels_from_pod_IP.sh" "$dn_IP")" + nodePartitionGroup="$(echo "$nodeLabels" | jq -r ".$partition_group_label")" + if [[ "$nodePartitionGroup" == "null" ]]; + then + nodeAZ="$(echo "$nodeLabels" | jq -r '."topology.kubernetes.io/zone"')" + if [[ "$nodeAZ" == "null" ]]; + then + rack="/default-rack" # when no partition group or availability zone info is found for the datanode + log "No partition groups or availability zones found; output default rack $rack for $dn_IP" + echo "$rack" + else + rack="/availability-zone-$nodeAZ" + log "output rack $rack for $dn_IP" + echo "$rack" + fi + else + rack="/partition-group-$nodePartitionGroup" + log "output rack $rack for $dn_IP" + echo "$rack" + fi +done diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/ssl-client.xml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/ssl-client.xml new file mode 100644 index 0000000000..3a8fffffb9 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/ssl-client.xml @@ -0,0 +1,52 @@ + + + + + ssl.client.keystore.keypassword + changeit + + + ssl.client.keystore.location + /tmp/scratch/keystore.jks + + + ssl.client.keystore.password + changeit + + + ssl.client.keystore.type + jks + + + ssl.client.truststore.location + /tmp/scratch/keystore.jks + + + ssl.client.truststore.password + changeit + + + ssl.client.truststore.reload.interval + 10000 + + + ssl.client.truststore.type + jks + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/ssl-server.xml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/ssl-server.xml new file mode 100644 index 0000000000..25e26ddfd8 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/base/ssl-server.xml @@ -0,0 +1,56 @@ + + + + + ssl.server.keystore.keypassword + + + + ssl.server.keystore.password + changeit + + + ssl.server.keystore.location + /tmp/scratch/keystore.jks + + + ssl.server.keystore.type + jks + + + ssl.server.truststore.location + /tmp/scratch/truststore.jks + + + ssl.server.truststore.keypassword + + + + ssl.server.truststore.password + changeit + + + ssl.server.truststore.reload.interval + 10000 + + + ssl.server.truststore.type + jks + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/kustomization.yaml new file mode 100644 index 0000000000..3cd62a711a --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/kustomization.yaml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Like hdfs except that region server runs as another container in the datanode pod. This arrangemt +# enables short-circuit reads between these processes. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: hadoop + +resources: +- ../hdfs + +components: +- ../../components/zookeeper/single-instance +- ../../components/hbase + +patches: +- # delete the region server Service + target: + kind: Service + name: regionserver + path: patches/regionserver_service_delete.yaml +- # delete the region server StatefulSet + target: + kind: StatefulSet + name: regionserver + path: patches/regionserver_statefulset_delete.yaml +- # introduce the short-circuit read volume to the datanode pod + target: + kind: StatefulSet + name: datanode + path: patches/datanode_statefulset_scr_volume.yaml +- # introduce the short-circuit read mount to the datanode container + target: + kind: StatefulSet + name: datanode + path: patches/datanode_statefulset_scr_datanode_mount.yaml +- # introduce the region server machinery into the datanode pod + target: + kind: StatefulSet + name: datanode + path: patches/datanode_regionserver_container.yaml +- # introduce the short-circuit read mount to the regionserver container + target: + kind: StatefulSet + name: datanode + path: patches/datanode_statefulset_scr_regionserver_mount.yaml +- # introduce the init container prepares the scr directory + target: + kind: StatefulSet + name: datanode + path: patches/datanode_statefulset_scr_initcontainer.yaml +- # introduce the init container that copies the hadoop native libs into the hbase container + target: + kind: StatefulSet + name: datanode + path: patches/datanode_statefulset_copy_native_initcontainer.yaml +- # include the hbase volumes in the datanode + target: + kind: StatefulSet + name: datanode + path: patches/datanode_statefulset_hbase_volumes.yaml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_regionserver_container.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_regionserver_container.yaml new file mode 100644 index 0000000000..6c6d0cbcf8 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_regionserver_container.yaml @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Introduce the region server container into the datanode pod +- op: add + path: /spec/template/spec/containers/- + value: + # this object is copied directly out of components/hbase/rs-statefulset.yaml. Changes there + # must be reflected here. + # TODO: find a better way to manage this shared blob of resource definition. Perhaps the + # cross-document anchors described in this blogh post will work? + # https://batalin.dev/posts/yaml-anchors-and-aliases/#reusing-yaml-anchors-between-multiple-files + image: hbase + name: regionserver + command: + - /bin/bash + - -c + - |- + # Shell context so we can pull in the environment variables set in the container and + # via the env and envFrom. + # + # See https://stackoverflow.com/questions/57885828/netty-cannot-access-class-jdk-internal-misc-unsafe + HBASE_CLASSPATH=/etc/hadoop \ + HBASE_OPTS=" \ + -XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -javaagent:${JMX_PROMETHEUS_JAR}=7000:/tmp/scratch/jmxexporter.yaml \ + -Djava.security.properties=/tmp/scratch/java.security \ + -Djava.library.path=/var/lib/hadoop-native-lib/native \ + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ + -Djava.util.logging.config.file=${HBASE_CONF_DIR}/logging.properties \ + -Dio.netty.tryReflectionSetAccessible=true \ + -Xlog:gc:${HBASE_LOG_DIR}/gc.log:time,uptime:filecount=10,filesize=100M" \ + hbase-daemon.sh --config /etc/hbase foreground_start regionserver \ + ${FAILOVER_PROXY_PROVIDER} # &> ${HBASE_LOG_DIR}/regionserver.$(date -u +"%Y-%m-%dT%H%M%SZ").out + # For now, just fetch local /jmx + # Says kubelet only exposes failures, not success: https://stackoverflow.com/questions/34455040/kubernetes-liveness-probe-logging + # TODO: Use https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/HealthChecker.html + livenessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16030 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16030 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + startupProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16030 + initialDelaySeconds: 10 + failureThreshold: 30 + periodSeconds: 10 + resources: + requests: + memory: "2Gi" + cpu: "0.2" + limits: + memory: "3Gi" + cpu: "1.0" + envFrom: + - configMapRef: + name: environment + - configMapRef: + name: zookeeper-quorum + - configMapRef: + name: jaegertracing + optional: true + - configMapRef: + name: xraytracing + optional: true + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 16030 + name: http + - containerPort: 16020 + name: rpc + volumeMounts: + - name: hbase-configuration + mountPath: /etc/hbase + readOnly: true + - name: hbck2-configuration + mountPath: /etc/hbck2 + readOnly: true + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hbase + name: hbase-logs + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/scripts + name: scripts + - mountPath: /var/lib/hadoop-native-lib + name: hadoop-native-lib +# - name: scr +# mountPath: /var/run/scr diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_copy_native_initcontainer.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_copy_native_initcontainer.yaml new file mode 100644 index 0000000000..c8124883cf --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_copy_native_initcontainer.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# introduce the init container that copies the hadoop native libs into the hbase container +- op: add + path: /spec/template/spec/volumes/- + value: + name: hadoop-native-lib + emptyDir: {} +- op: add + path: /spec/template/spec/initContainers/- + value: + image: hadoop + name: copy-hadoop-native-lib + command: + # Runs as the image/hdfs user. + - /bin/bash + - -c + - |- + set -xe + if [ -d ~/hadoop/lib/native ]; then + cp -r ~/hadoop/lib/native /var/lib/hadoop-native-lib + else + echo "Native dir not found at ~/hadoop/lib/native" + fi + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + volumeMounts: + - mountPath: /var/lib/hadoop-native-lib + name: hadoop-native-lib diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_hbase_volumes.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_hbase_volumes.yaml new file mode 100644 index 0000000000..139a21b015 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_hbase_volumes.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# include the hbase ConfgMaps in the datanode +- op: add + path: /spec/template/spec/volumes/- + value: + name: hbase-configuration + configMap: + name: hbase-configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: hbck2-configuration + configMap: + name: hbck2-configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: hbase-logs + emptyDir: {} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_bootstrapper_mount.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_bootstrapper_mount.yaml new file mode 100644 index 0000000000..84bbd71599 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_bootstrapper_mount.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# introduce the short-circuit read mount to the datanode container +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +spec: + template: + spec: + initContainers: + - name: bootstrapper + volumeMounts: + - name: scr + mountPath: /var/run/scr diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_datanode_mount.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_datanode_mount.yaml new file mode 100644 index 0000000000..34db0f547d --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_datanode_mount.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# introduce the short-circuit read mount to the datanode container +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +spec: + template: + spec: + containers: + - name: datanode + volumeMounts: + - name: scr + mountPath: /var/run/scr diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_initcontainer.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_initcontainer.yaml new file mode 100644 index 0000000000..0de861d6cf --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_initcontainer.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# introduce the init container prepares the scr directory +- op: add + path: /spec/template/spec/initContainers/- + value: + image: hadoop + name: scr-bootstrapper + command: + - /bin/bash + - -c + - |- + set -xe + scr_dir="/var/run/scr" + ls -lad ${scr_dir} || $? + chown -R ${NON_ROOT_USER} ${scr_dir} || echo $? + chmod -R 700 ${scr_dir} || echo $? + ls -lad ${scr_dir} || $? + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + envFrom: + - configMapRef: + name: environment + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/scr + name: scr diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_regionserver_mount.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_regionserver_mount.yaml new file mode 100644 index 0000000000..761d9b54de --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_regionserver_mount.yaml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# introduce the short-circuit read mount to the regionserver container +- op: add + path: /spec/template/spec/containers/1/volumeMounts/- + value: + name: scr + mountPath: /var/run/scr diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_volume.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_volume.yaml new file mode 100644 index 0000000000..05453ff70b --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/datanode_statefulset_scr_volume.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# introduce the short-circuit read volume to the datanode pod +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +spec: + template: + spec: + volumes: + - name: scr + emptyDir: {} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/regionserver_service_delete.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/regionserver_service_delete.yaml new file mode 100644 index 0000000000..7bd72c1bea --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/regionserver_service_delete.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# delete the region server Service +kind: Service +metadata: + name: regionserver +$patch: delete diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/regionserver_statefulset_delete.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/regionserver_statefulset_delete.yaml new file mode 100644 index 0000000000..5bd5d9a905 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs+hbase-colocated/patches/regionserver_statefulset_delete.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# introduce the short-circuit read volume to the datanode pod +kind: StatefulSet +metadata: + name: regionserver +$patch: delete diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/core-site.xml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/core-site.xml new file mode 100644 index 0000000000..782c61f0e9 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/core-site.xml @@ -0,0 +1,96 @@ + + + + + fs.defaultFS + hdfs://${env.HADOOP_SERVICE} + + + fs.trash.interval + 10080 + + + fs.trash.checkpoint.interval + 10080 + + + ha.zookeeper.acl + world:anyone:rwcda + + + ha.zookeeper.auth + + + + ha.zookeeper.quorum + ${env.HA_ZOOKEEPER_QUORUM} + + + ha.zookeeper.parent-znode + / + + + hadoop.proxyuser.hdfs.hosts + * + + + hadoop.proxyuser.hdfs.users + * + + + hadoop.user.group.static.mapping.overrides + hdfs=supergroup;nobody=; + + + net.topology.script.file.name + /tmp/scripts/topology.sh + + + net.topology.script.number.args + 1 + + + hadoop.rpc.protection + authentication + + + hadoop.security.authorization + false + + + hadoop.ssl.client.conf + ssl-client.xml + + + hadoop.ssl.enabled + false + + + hadoop.ssl.keystores.factory.class + org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory + + + hadoop.ssl.require.client.cert + false + + + hadoop.ssl.server.conf + ssl-server.xml + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/dn-service.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/dn-service.yaml new file mode 100644 index 0000000000..41515276a2 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/dn-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +kind: Service +apiVersion: v1 +metadata: + name: datanode + labels: + jmxexporter: enabled +spec: + selector: + role: datanode + clusterIP: None + ports: + - name: jmxexporter + port: 8000 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/dn-statefulset.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/dn-statefulset.yaml new file mode 100644 index 0000000000..b03640dcab --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/dn-statefulset.yaml @@ -0,0 +1,222 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + role: datanode + serviceName: hadoop + template: + metadata: + labels: + role: datanode + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + role: datanode + topologyKey: kubernetes.io/hostname + containers: + - image: hadoop + name: datanode + command: + - /bin/bash + - -c + - |- + # Shell context so we can pull in the environment variables set in the container and + # via the env and envFrom. + # See https://stackoverflow.com/questions/57885828/netty-cannot-access-class-jdk-internal-misc-unsafe + HADOOP_LOGFILE="hdfs-${HOSTNAME}.log" \ + HDFS_DATANODE_OPTS=" \ + -XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -Djava.security.properties=/tmp/scratch/java.security \ + -javaagent:${JMX_PROMETHEUS_JAR}=8000:/tmp/scratch/jmxexporter.yaml \ + -Djava.library.path=${HADOOP_HOME}/lib/native \ + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ + -Dio.netty.tryReflectionSetAccessible=true \ + -Xlog:gc:/var/log/hadoop/gc.log:time,uptime:filecount=10,filesize=100M" \ + hdfs datanode + # For now, just fetch local /jmx + # Says kubelet only exposes failures, not success: https://stackoverflow.com/questions/34455040/kubernetes-liveness-probe-logging + # Do better. Check this DN successfully registered w/ NN. TODO. + livenessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + # 9865 if HTTPS + port: 9864 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + # 9865 if HTTPS + port: 9864 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + startupProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + # 9865 if HTTPS + port: 9864 + initialDelaySeconds: 10 + failureThreshold: 30 + periodSeconds: 10 + resources: + requests: + cpu: '0.2' + memory: 1Gi + limits: + cpu: '1.0' + memory: 1.5Gi + envFrom: + - configMapRef: + name: environment + env: + # The 'node' this container is running on, not hdfs namenode. + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 9864 + - name: https + containerPort: 9865 + - name: data + containerPort: 9866 + - name: ipc + containerPort: 9867 + - name: jmx + containerPort: 9864 + volumeMounts: + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hadoop + name: hadoop-logs + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/scripts + name: scripts + - mountPath: /data00 + name: data00 + initContainers: + - image: hadoop + name: bootstrapper + command: + - /bin/bash + - -c + - |- + set -xe + mkdir -p ${HADOOP_LOG_DIR} || echo $? + chown -R ${USER} ${HADOOP_LOG_DIR} + # If format-hdfs configmap present, format. + ! /tmp/scripts/exists_configmap.sh format-hdfs || ( + for dir in $( echo "${DATANODE_DATA_DIR}" | tr ',' '\n') + do + rm -rf ${dir} + done + ) + for dir in $( echo "${DATANODE_DATA_DIR}" | tr ',' '\n') + do + mkdir -p ${dir} || : + chown -R ${USER} ${dir} + done + df -h + cp /tmp/global-files/* /tmp/scratch/ + # Wait for the nns to come up. + /tmp/scripts/jmxping.sh namenode ${HADOOP_SERVICE} + securityContext: + # Run bootstrapper as root so can set ${USER} owner on data volume + allowPrivilegeEscalation: false + runAsUser: 0 + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + env: + # Used by scripts that run during bootstrap + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /data00 + name: data00 + - mountPath: /tmp/scripts + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/global-files + name: global-files + serviceAccountName: hadoop + volumes: + - configMap: + name: hadoop-configuration + name: hadoop-configuration + - configMap: + name: scripts + defaultMode: 0555 + name: scripts + - configMap: + name: global-files + name: global-files + - emptyDir: {} + name: hadoop-logs + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - emptyDir: {} + name: scratch + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - metadata: + name: data00 + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/hdfs-site.xml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/hdfs-site.xml new file mode 100644 index 0000000000..2ae424942e --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/hdfs-site.xml @@ -0,0 +1,274 @@ + + + + + dfs.block.replicator.classname + org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant + + + dfs.blocksize + 64m + + + dfs.datanode.address + 0.0.0.0:9866 + + + dfs.datanode.balance.bandwidthPerSec + 20m + + + dfs.datanode.balance.max.concurrent.moves + 100 + + + dfs.datanode.data.dir + ${env.DATANODE_DATA_DIR} + + + dfs.datanode.failed.volumes.tolerated + 0 + + + dfs.datanode.du.reserved + 1073741824 + + + dfs.datanode.fileio.profiling.sampling.percentage + 10 + + + dfs.datanode.http.address + 0.0.0.0:9864 + + + dfs.datanode.https.address + 0.0.0.0:9865 + + + dfs.datanode.ipc.address + 0.0.0.0:9867 + + + dfs.datanode.max.locked.memory + 0 + + + dfs.datanode.peer.stats.enabled + true + + + dfs.encrypt.data.transfer + false + + + dfs.encrypt.data.transfer.algorithm + rc4 + + + dfs.ha.automatic-failover.enabled + true + + + dfs.ha.fencing.methods + shell(/usr/bin/true) + + + dfs.journalnode.edits.dir + ${env.JOURNALNODE_DATA_DIR} + + + dfs.journalnode.http-address + 0.0.0.0:8480 + + + dfs.journalnode.https-address + 0.0.0.0:8481 + + + dfs.journalnode.rpc-address + 0.0.0.0:8485 + + + dfs.namenode.handler.count + 64 + + + + + dfs.namenode.http-bind-host + 0.0.0.0 + + + dfs.namenode.https-bind-host + 0.0.0.0 + + + dfs.namenode.name.dir + ${env.NAMENODE_DATA_DIR} + + + dfs.namenode.replication.max-streams + 20 + + + dfs.namenode.replication.max-streams-hard-limit + 40 + + + dfs.namenode.replication.min + ${env.DFS_REPLICATION} + + + dfs.namenode.replication.work.multiplier.per.iteration + 10 + + + dfs.namenode.safemode.threshold-pct + 0.9 + + + dfs.namenode.service.handler.count + 64 + + + dfs.nameservices + ${env.HADOOP_SERVICE} + + + dfs.reformat.disabled + false + + + dfs.replication + ${env.DFS_REPLICATION} + + + dfs.replication.max + 512 + + + ipc.8020.callqueue.impl + org.apache.hadoop.ipc.FairCallQueue + + + ipc.8020.scheduler.impl + org.apache.hadoop.ipc.DecayRpcScheduler + + + zk-dt-secret-manager.zkAuthType + digest + + + zk-dt-secret-manager.digest.auth + @/etc/hadoop/zookeeper/auth/zk-auth.txt + + + zk-dt-secret-manager.zkConnectionString + TODO + + + zk-dt-secret-manager.znodeWorkingPath + TODO + + + dfs.client.failover.proxy.provider.hadoop + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + + dfs.ha.namenodes.hadoop + namenode-0 + + + dfs.namenode.http-address.hadoop.namenode-0 + namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9870 + + + dfs.namenode.https-address.hadoop.namenode-0 + namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9871 + + + dfs.namenode.rpc-address.hadoop.namenode-0 + namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8020 + + + dfs.namenode.servicerpc-address.hadoop.namenode-0 + namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8022 + + + dfs.namenode.lifeline.rpc-address.hadoop.namenode-0 + namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8050 + + + dfs.client.https.keystore.resource + ssl-client.xml + + + dfs.client.https.need-auth + false + + + dfs.http.policy + ${env.HTTP_POLICY} + + + dfs.https.enable + ${env.DFS_HTTPS_ENABLE} + + + dfs.https.server.keystore.resource + ssl-server.xml + + + dfs.namenode.acls.enabled + true + + + dfs.datanode.use.datanode.hostname + true + + + dfs.client.use.datanode.hostname + true + + + + dfs.namenode.datanode.registration.ip-hostname-check + false + + + dfs.blockreport.intervalMsec + 900000 + Determines block reporting interval in milliseconds. + Report frequently else around recovery storms, the NN gets convinced + there is no block space left because of 'scheduled space' reserved. + + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/kustomization.yaml new file mode 100644 index 0000000000..6bd3abea34 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/kustomization.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: hadoop + +configMapGenerator: +- name: hadoop-configuration + # Add in single-instance namenode and datanode hdfs-site and core-site. + behavior: merge + files: + - hdfs-site.xml + - core-site.xml + +resources: +- nn-statefulset.yaml +- nn-service.yaml +- dn-statefulset.yaml +- dn-service.yaml +- ../../base diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/nn-service.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/nn-service.yaml new file mode 100644 index 0000000000..a46b448c08 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/nn-service.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +kind: Service +apiVersion: v1 +metadata: + name: namenode + labels: + jmxexporter: enabled +spec: + selector: + role: namenode + clusterIP: None + ports: + - name: rpc + port: 8020 + protocol: TCP + targetPort: 8020 + - name: http + port: 9870 + protocol: TCP + targetPort: 9870 + - name: jmxexporter + port: 8000 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/nn-statefulset.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/nn-statefulset.yaml new file mode 100644 index 0000000000..789e46458d --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/overlays/hdfs/nn-statefulset.yaml @@ -0,0 +1,326 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: namenode +spec: + minAvailable: 1 + selector: + matchLabels: + role: namenode +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: namenode +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + role: namenode + serviceName: hadoop + template: + metadata: + labels: + role: namenode + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: role + operator: In + values: + - namenode + topologyKey: kubernetes.io/hostname + containers: + - image: hadoop + name: namenode + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + - |- + # Shell context so we can pull in the environment variables set in the container and + # via the env and envFrom. + # See https://stackoverflow.com/questions/57885828/netty-cannot-access-class-jdk-internal-misc-unsafe + HADOOP_LOGFILE="hdfs-${HOSTNAME}.log" \ + HDFS_NAMENODE_OPTS=" \ + -XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -Djava.security.properties=/tmp/scratch/java.security \ + -javaagent:${JMX_PROMETHEUS_JAR}=8000:/tmp/scratch/jmxexporter.yaml \ + -Djava.library.path=${HADOOP_HOME}/lib/native \ + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ + -Dio.netty.tryReflectionSetAccessible=true \ + -Xlog:gc:/var/log/hadoop/gc.log:time,uptime:filecount=10,filesize=100M" \ + hdfs namenode + # For now, just fetch local /jmx + # Says kubelet only exposes failures, not success: https://stackoverflow.com/questions/34455040/kubernetes-liveness-probe-logging + livenessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + # 9871 if HTTPS + port: 9870 + initialDelaySeconds: 1 + failureThreshold: 6 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + # 9871 if HTTPS + port: 9870 + initialDelaySeconds: 10 + failureThreshold: 3 + periodSeconds: 10 + startupProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + # 9871 if HTTPS + port: 9870 + initialDelaySeconds: 10 + failureThreshold: 30 + periodSeconds: 10 + resources: + requests: + cpu: '0.4' + memory: 2Gi + limits: + cpu: '1' + memory: 3Gi + envFrom: + - configMapRef: + name: environment + env: + # The 'node' this container is running on, not hdfs namenode. + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 9870 + - name: https + containerPort: 9871 + - name: jmx + containerPort: 9870 + - name: rpc + containerPort: 8020 + - name: servicerpc + containerPort: 8022 + - name: lifelinerpc + containerPort: 8050 + volumeMounts: + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hadoop + name: hadoop-logs + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/scripts + name: scripts + - mountPath: /data00 + name: data00 + initContainers: + - image: hadoop + name: bootstrapper + imagePullPolicy: IfNotPresent + command: + # This container is running as root so can set permissions. + - /bin/bash + - -c + - |- + set -xe + if [ -n "${QJOURNAL}" ]; then + # If QJOURNAL, then HA and journalnodes are in the mix. Wait on them to come up. + /tmp/scripts/jmxping.sh journalnode ${HADOOP_SERVICE} + fi + # Copy over the files under global-files so in place for the runtime container. + cp /tmp/global-files/* /tmp/scratch/ + # Set perms + chown -R ${USER} ${HADOOP_LOG_DIR} + # If format-hdfs configmap present, format. + find ${NAMENODE_DATA_DIR} || : + ! /tmp/scripts/exists_configmap.sh format-hdfs || ( + rm -rf ${NAMENODE_DATA_DIR} + ) + chmod 777 /data00 + securityContext: + # Run bootstrapper as root so can set ${USER} owner on data volume + allowPrivilegeEscalation: false + runAsUser: 0 + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hadoop + name: hadoop-logs + - mountPath: /data00 + name: data00 + - mountPath: /etc/hadoop/zookeeper/auth + name: zookeeper-credentials + readOnly: true + - mountPath: /tmp/scripts + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/global-files + name: global-files + - image: hadoop + name: format-hdfs + imagePullPolicy: IfNotPresent + command: + # Runs as the image/hdfs user. + - /bin/bash + - -c + - |- + set -xe + find /data00 || echo $? + # Run format if no nn dir. + if [ ! -d "${NAMENODE_DATA_DIR}" ]; then + ordinal=$(echo $POD_NAME | sed -e 's/^[^-]*-\(.*\)/\1/') + case $ordinal in + 0) + hdfs namenode -format -nonInteractive || ( + # Perhaps another nn is active? If so, we should do bootstrap here instead. + hdfs namenode -bootstrapStandby -nonInteractive + ) + ;; + *) + hdfs namenode -bootstrapStandby -nonInteractive + ;; + esac + fi + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hadoop + name: hadoop-logs + - mountPath: /data00 + name: data00 + - mountPath: /etc/hadoop/zookeeper/auth + name: zookeeper-credentials + readOnly: true + - mountPath: /tmp/scripts + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - mountPath: /tmp/scratch + name: scratch + serviceAccountName: hadoop + volumes: + - configMap: + name: hadoop-configuration + name: hadoop-configuration + - configMap: + name: scripts + defaultMode: 0555 + name: scripts + - configMap: + name: global-files + name: global-files + - emptyDir: {} + name: hadoop-logs + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - emptyDir: {} + name: scratch + - secret: + secretName: zookeeper-credentials + defaultMode: 400 + optional: true + name: zookeeper-credentials + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - metadata: + name: data00 + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/README.md new file mode 100644 index 0000000000..69a91c23ee --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/README.md @@ -0,0 +1,159 @@ + + +# Kubernetes Deployment Testing Using + +Defines a set of tests that are suitable for running against a target cluster -- they are not too +resource intensive and do not require any vendor-specific extensions. It should be possible to run +these tests against a multi-node KinD cluster, below are some notes to help a developer to run +them locally. + +## Run the tests locally + +Assumes a Docker Desktop or some other docker-in-docker type of environment. First, prepare your +cluster connection details such that they can be passed into the container context. Next, launch +the test runner in a container: + +```shell +$ docker container run \ + --env KUBECONFIG=/workspace/your-kubeconfig \ + --mount type=bind,source=$(PWD),target=/workspace \ + -v /var/run/docker.sock:/var/run/docker.sock \ + --workdir /workspace \ + ${USER}/hbase/operator-tools/kuttl:latest \ + --config tests/kuttl-test-integration.yaml \ + --parallel 1 +``` + +## Run the tests in AWS EKS + +It is possible to run these tests in AWS EKS. This requires configuring an RBAC on your target +cluster that maps to an AIM profile. Next, define a profile in AWS configuration. When you launch +the container, pass configuration and profile selection through to the running container. + +Building on the previous example, + +```shell +$ docker container run \ + --env AWS_PROFILE="your-profile" \ + --env KUBECONFIG=/workspace/your-kubeconfig \ + --mount type=bind,source=$(PWD),target=/workspace \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ~/.aws:/root/.aws \ + --workdir /workspace \ + ${USER}/hbase/operator-tools/kuttl:latest \ + --config tests/kuttl-test-integration.yaml +``` + +## Prepare a KinD cluster + +Ask KinD to create a cluster (and docker network), and export the configuration oriented as from +inside the cluster. Start by creating a kind-config.yaml and configuring it for muliple nodes. +See https://kind.sigs.k8s.io/docs/user/quick-start/#configuring-your-kind-cluster + +```shell +$ kind create cluster --config kind-config.yaml +... +You can now use your cluster with: + +kubectl cluster-info --context kind --kubeconfig kubeconfig +$ kind export kubeconfig --name kind --internal --kubeconfig kubeconfig-internal +``` + +## Local KinD Hacks + +Preparing and staging the large container images into the kind nodes is slow. Speed up the process +a bit by creating a single-node KinD cluster and letting `kuttl` populate the images you need. + +First, find all the images used in your tests, + +```shell +$ find tests/kind -type f -iname '*kustomization.yaml' \ + -exec yq '.images[] | .newName + ":" + .newTag' {} + \ + | sort -u +hadoop:... +hbase:... +zookeeper:... +``` + +Pull those images locally. + +```shell +$ docker image pull hadoop:... +$ docker image pull hbase:... +$ docker image pull zookeeper:... +``` + +Now make sure kuttl is using a docker volume for the containerd directory on each container, and +populate those images into your kuttl configuration using this config snippet: + +```yaml +kindNodeCache: + # Have kuttl create and mount volumes for a container image cache to each kind pod. Kuttl will + # reuse these mounts across runs, so we can save time the next the tests run. + true +kindContainers: + # pre-populate the kind containers with these images pulled from the host registry. They'll be + # cached via `kindNodeCache`. +- hadoop... +- hbase... +- zookeeper:... +``` + +When you run `kuttl` with this config, you'll see that it has mounted a volume for each container. +It'll take a while, but `kuttl` will report its progress copying these container images. + +``` +== RUN kuttl +... + harness.go:202: node mount point /var/lib/docker/volumes/kind-0/_data +... + harness.go:155: Starting KIND cluster + kind.go:66: Adding Containers to KIND... + kind.go:75: Add image zookeeper:... to node control-plane +... +``` + +Once copied into one volume, create all the additional volumes you'll need and clone the original. +Repeat this for every worker node you'd like in your cluster. + +```shell +$ docker volume create --name kind-1 +$ docker container run --rm -it \ + -v kind-0:/from \ + -v kind-1:/to \ + alpine ash -c "cd /from ; cp -a . /to" +``` + +In `kind-config.yaml`, specify the mount points for each of your KinD processes. + +```yaml +nodes: +- role: control-plane + extraMounts: + - &extra-mounts + hostPath: /var/lib/docker/volumes/kind-0/_data + containerPath: /var/lib/containerd + readOnly: false + propagation: HostToContainer +- role: worker + extraMounts: + - <<: *extra-mounts + hostPath: /var/lib/docker/volumes/kind-1/_data +... +``` diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/00-assert.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/00-assert.yaml new file mode 100644 index 0000000000..32e31d5437 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/00-assert.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# assert that there is a `StatefulSet` named "namenode" that has one live instance +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: namenode +status: + availableReplicas: 1 +--- +# assert that there is a `StatefulSet` named "datanode" that has one live instance +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +status: + availableReplicas: 1 +--- +# assert that there is a `StatefulSet` named "master" that has one live instance +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: master +status: + availableReplicas: 1 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/00-kustomize.yaml new file mode 100644 index 0000000000..3947f48c77 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/00-kustomize.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: ../../bin/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/kustomization.yaml new file mode 100644 index 0000000000..2367ae30f7 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs+hbase-colocated/kustomization.yaml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../test_base +- ../../../overlays/hdfs+hbase-colocated diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/00-assert.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/00-assert.yaml new file mode 100644 index 0000000000..899c485303 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/00-assert.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# assert that there is a `StatefulSet` named "namenode" that has one live instance +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: namenode +status: + availableReplicas: 1 +--- +# assert that there is a `StatefulSet` named "datanode" that has one live instance +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +status: + availableReplicas: 1 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/00-kustomize.yaml new file mode 100644 index 0000000000..3947f48c77 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/00-kustomize.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: ../../bin/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/kustomization.yaml new file mode 100644 index 0000000000..ba9a7bf866 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/overlays_hdfs/kustomization.yaml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../test_base +- ../../../overlays/hdfs diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/test_base/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/test_base/kustomization.yaml new file mode 100644 index 0000000000..387cf31eeb --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/test_base/kustomization.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + # Must repeat common labels and images in each overlay; can't inherit to keep each overlay independent + # https://github.com/kubernetes-sigs/kustomize/issues/915 + # This label is used to open up calico network acls + app: hadoop + +resources: +# When inter-pod networking is limited, apply this policy to open communications between pods that +# bear the "hadoop" label. +- networkpolicy.yaml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/test_base/networkpolicy.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/test_base/networkpolicy.yaml new file mode 100644 index 0000000000..8da0c53f70 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/test_base/networkpolicy.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Explicitly permit all traffic between Hadoop-related pods in our namespace +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-all +spec: + podSelector: + matchLabels: + app: hadoop + ingress: + - {} + egress: + - {} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/00-assert.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/00-assert.yaml new file mode 100644 index 0000000000..1e24de4eb8 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/00-assert.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# assert that there is a `ConfigMap` named "environment" +apiVersion: v1 +kind: ConfigMap +metadata: + name: environment +--- +# assert that there is a `ConfigMap` named "global-files" +apiVersion: v1 +kind: ConfigMap +metadata: + name: global-files +# TODO: kuttl has no means to express `any` value, so cannot assert on data keys. +#data: +# java.security: ... +# jmxexporter.yaml: ... +# ssl-client.xml: ... +# ssl-server.xml: ... +--- +# assert that there is a `ConfigMap` named "hadoop-configuration-XXX" +# TODO: kuttl does not support generated names +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: hadoop-configuration-c94h8k249d +# TODO: kuttl has no means to express `any` value, so cannot assert on data keys. +#data: +# log4j.properties: ... +--- +# assert that there is a `ConfigMap` named "scripts" +apiVersion: v1 +kind: ConfigMap +metadata: + name: scripts +# TODO: kuttl has no means to express `any` value, so cannot assert on data keys. +#data: +# apiserver_access.sh: ... +# ... +--- +# assert that there is a `Secret` named "keystore-password" +apiVersion: v1 +kind: Secret +metadata: + name: keystore-password +type: Opaque +--- +# assert that there is a `Service` names "hadoop" +apiVersion: v1 +kind: Service +metadata: + name: hadoop +--- +# assert that there is a `Job` named "delete-format-hdfs-configmap" +apiVersion: batch/v1 +kind: Job +metadata: + name: delete-format-hdfs-configmap diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/00-kustomize.yaml new file mode 100644 index 0000000000..c42afed213 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/00-kustomize.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: ../../../resources/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/README.md new file mode 100644 index 0000000000..e6a4aef607 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/README.md @@ -0,0 +1,24 @@ + + +# tests/unit/base + +A collection of asserts on the resources allocated by `hbase-kubernetes-deployment/base` that are +not explicitly covered by a more specific test case. + +Hopefully the scope of this test case shrinks over time. diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/kustomization.yaml new file mode 100644 index 0000000000..82e3b2c241 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/base/kustomization.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../main/kustomize/base diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-assert-hbase.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-assert-hbase.yaml new file mode 100644 index 0000000000..203eb7f08f --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-assert-hbase.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the HBase portion of the deployment. +# +--- +# assert that there is a `StatefulSet` named "master" that it provides pods labeled role:master +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: master +spec: + template: + metadata: + labels: + role: master +--- +# assert that there is a `Service` named "master" pointing to pods labeled role:master +apiVersion: v1 +kind: Service +metadata: + name: master +spec: + selector: + role: master diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-assert-hdfs.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-assert-hdfs.yaml new file mode 100644 index 0000000000..2565dc4e30 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-assert-hdfs.yaml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the HDFS portion of the deployment. +# +--- +# assert that there is a `StatefulSet` named "datanode" that: +# - provides pods labeled role:datanode +# - has containers named 'datanode' and 'hbase' +# - has a "scr" volume attached +# - has an initContainer that prepares the scr mount +# - the datanode and hbase containers have a short-circuit reads volume mounted +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +spec: + template: + spec: + containers: + - name: datanode + volumeMounts: + - name: scr + - name: hadoop-configuration + - name: hadoop-logs + - name: scratch + - name: scripts + - name: data00 + - name: regionserver + volumeMounts: + - name: hbase-configuration + - name: hbck2-configuration + - name: hadoop-configuration + - name: hbase-logs + - name: scratch + - name: scripts + - name: hadoop-native-lib + - name: scr + initContainers: + - name: bootstrapper + - name: scr-bootstrapper + - name: copy-hadoop-native-lib + volumes: + - name: scr + - name: hadoop-configuration + - name: scripts + - name: global-files + - name: hadoop-logs + - name: scratch + - name: hadoop-native-lib + - name: hbase-configuration + - name: hbck2-configuration + - name: hbase-logs diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-errors-hbase.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-errors-hbase.yaml new file mode 100644 index 0000000000..583ace2bcf --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-errors-hbase.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The presence of anything below will result in an error +# +--- +# assert that there is NOT a `StatefulSet` named "regionserver" +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: regionserver +spec: + template: + metadata: + labels: + role: regionserver +--- +# assert that there is NOT a `Service` named "regionserver" +apiVersion: v1 +kind: Service +metadata: + name: regionserver diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-kustomize.yaml new file mode 100644 index 0000000000..3947f48c77 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/00-kustomize.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: ../../bin/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/kustomization.yaml new file mode 100644 index 0000000000..ea7d98992b --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs+hbase-colocated/kustomization.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../overlays/hdfs+hbase-colocated diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/00-assert.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/00-assert.yaml new file mode 100644 index 0000000000..cf367108c6 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/00-assert.yaml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# assert that there is a `ConfigMap` named "hadoop-configuration-XXX" +# TODO: kuttl does not support generated names +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: hadoop-configuration-c94h8k249d +# TODO: kuttl has no means to express `any` value, so cannot assert on data keys. +#data: +# log4j.properties: ... +# hdfs-site.xml: ... +# core-site.xml: ... +--- +# assert that there is a `PodDisruptionBudget` named "namenode" +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: namenode +--- +# assert that there is a `StatefulSet` named "namenode" that it provides pods labeled role:namenode +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: namenode +spec: + template: + metadata: + labels: + role: namenode +--- +# assert that there is a `Service` named "namenode" pointing to pods labeled role:namenode +apiVersion: v1 +kind: Service +metadata: + name: namenode +spec: + selector: + role: namenode +--- +# assert that there is a `StatefulSet` named "datanode" that it provides pods labeled role:datanode +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode +spec: + template: + metadata: + labels: + role: datanode +--- +# assert that there is a `Service` named "datanode" pointing to pods labeled role:datanode +apiVersion: v1 +kind: Service +metadata: + name: datanode +spec: + selector: + role: datanode diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/00-kustomize.yaml new file mode 100644 index 0000000000..c42afed213 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/00-kustomize.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: ../../../resources/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/kustomization.yaml new file mode 100644 index 0000000000..a557e53e95 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/overlays_hdfs/kustomization.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../../main/kustomize/overlays/hdfs diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh new file mode 100755 index 0000000000..7e5cef1048 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# +# Materialize a kustomize directory for a kuttl test. +# +# Kustomize is clunky for automated testing. It's pretty opinionated in that it will only evaluate +# a directory off of disk -- you cannot generate a kustomization and pass it in via stdin. +# In order to use kuttl generated namespaces within the kustomization, we have to modify the +# kustomization.yaml before applying it. If we modify that file in the source tree, we end up with +# the test namespace appended to the file under source control. So, this script creates a temp +# directory, copies all the resources into that directory, and modifies the kustomization.yaml as +# necessary. It then runs `kubectl apply -k` against that temporary directory. +# + +declare DEBUG="${DEBUG:false}" +if [ "${DEBUG}" = 'true' ] ; then + set -x +fi + +set -eou pipefail + +declare NAMESPACE +declare NEW_RESOURCES='[]' +declare NEW_COMPONENTS='[]' +declare kustomize_dir +declare -a rewritten_resources=() +declare -a rewritten_components=() + +kustomize_dir="$(mktemp -d -p /tmp "${NAMESPACE}.XXXXXXXXXX")" +trap '[ -d "${kustomize_dir}" ] && rm -rf "${kustomize_dir}"' EXIT + +cp -r ./* "${kustomize_dir}/" + +for r in $(yq '.resources[]' kustomization.yaml) ; do + if [[ "${r}" =~ ^\.\./.* ]] ; then + # resolve the new relative location for any resource path that is not in the local directory + canonized="$(cd "${r}" ; pwd)" + r="../..${canonized}" + fi + rewritten_resources+=("'${r}'") +done +if [ "${#rewritten_resources[@]}" -gt 0 ] ; then + NEW_RESOURCES="[ $(printf '%s,' "${rewritten_resources[@]}") ]" +fi + +for r in $(yq '.components[]' kustomization.yaml) ; do + if [[ "${r}" =~ ^\.\./.* ]] ; then + # resolve the new relative location for any resource path that is not in the local directory + canonized="$(cd "${r}" ; pwd)" + r="../..${canonized}" + fi + rewritten_components+=("'${r}'") +done +if [ "${#rewritten_components[@]}" -gt 0 ] ; then + NEW_COMPONENTS="[ $(printf '%s,' "${rewritten_components[@]}") ]" +fi + +env NAMESPACE="${NAMESPACE}" \ + NEW_RESOURCES="${NEW_RESOURCES}" \ + NEW_COMPONENTS="${NEW_COMPONENTS}" \ + yq -i ' + .namespace = strenv(NAMESPACE) | + .resources = env(NEW_RESOURCES) | + .components = env(NEW_COMPONENTS) +' "${kustomize_dir}/kustomization.yaml" + +if [ "${DEBUG}" = 'true' ] ; then + cat "${kustomize_dir}/kustomization.yaml" +fi + +kubectl apply -k "${kustomize_dir}" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml new file mode 100644 index 0000000000..3329cdf10d --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +--- +# Does not use Kuttl's built-in KIND support -- it doesn't quite work correctly with a VM-based +# (Docker Desktop) style of runtime. Instead, assumes the cluster is established outside of kuttl +# and configuration is provided via `--env`. +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: +- ./src/test/kustomize/integration +timeout: + # these tests allocate several pods with dependencies between them, allow some time for + # everything to launch and settle. + 300 +reportName: TEST-kuttl-report-integration +reportFormat: xml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml new file mode 100644 index 0000000000..d87b2d6691 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# Test runner using https://kuttl.dev +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +startControlPlane: true +testDirs: +- ./src/test/kustomize/unit +reportName: TEST-kuttl-report-unit +reportFormat: xml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh new file mode 100755 index 0000000000..e544d65fac --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Wrap up the complexities of launching `kubectl kuttl test` via docker container. + +set -euox pipefail + +declare default_run_args +default_run_args="--rm --mount type=bind,source=$(pwd),target=/workspace --workdir /workspace" + +# from the calling environment +declare DOCKER_EXE="${DOCKER_EXE:-"$(command -v docker 2>/dev/null)"}" +declare DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS="${DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS:-"${default_run_args}"}" +declare USER="${USER:-apache}" +declare KUTTL_IMAGE="${KUTTL_IMAGE:-"${USER}/hbase/operator-tools/kuttl:latest"}" + +declare run_args +read -r -a run_args <<< "$DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS" + +exec "${DOCKER_EXE}" container run \ + "${run_args[@]}" \ + "${KUTTL_IMAGE}" \ + "$@" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/README.md new file mode 100644 index 0000000000..1dffc2a591 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/README.md @@ -0,0 +1,70 @@ + + +# hbase-kubernetes-testing-image + +This directory builds a docker image containing everything required to run `kubectl-kuttl` in +"mocked control plane" mode. This image is used as the basis for both dev and test environments. + +## Build + +Building the docker image locally is a little picky because there's lots of input arguments. These +are managed via the [docker-bake.override.hcl](./src/main/docker/docker-bake.override.hcl). + +Start by creating a buildx context that supports (optionally) multi-platform images. If you've +created this context previously, it's enough to ensure that it's active via `docker buildx ls`. + +```shell +$ docker buildx create \ + --driver docker-container \ + --platform linux/amd64,linux/arm64 \ + --use \ + --bootstrap +``` + +Finally, build the image using `maven package`, or manually, using, + +```shell +$ docker buildx bake \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl \ + --pull \ + --load +``` + +This exports an image to your local repository that is tagged as `${USER}/hbase/operator-tools/kuttl:latest`. + +## Usage + +The image is configured with `kuttle` as the entrypoint. + +```shell +$ docker container run --rm -it ${USER}/hbase/operator-tools/kuttl:latest --help + +``` + +Running tests in the image requires mounting the workspace into the container image and passing +appropriate parameters to `kuttl`. For example, run the "unit" tests like this: + +```shell +$ docker container run \ + --mount type=bind,source=$(pwd),target=/workspace \ + --workdir /workspace \ + ${USER}/hbase/operator-tools/kuttl:latest \ + --config tests/kuttl-test-unit.yaml +``` diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/pom.xml b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/pom.xml new file mode 100644 index 0000000000..f6b7c0e65a --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/pom.xml @@ -0,0 +1,126 @@ + + + + 4.0.0 + + hbase-kubernetes-deployment + org.apache.hbase.operator.tools + ${revision} + .. + + + hbase-kubernetes-testing-image + Apache HBase - Kubernetes Testing Image + A container image to facilitate testing of Kubernetes Deployment. + pom + + + linux/amd64 + + + + + + + + org.codehaus.mojo + exec-maven-plugin + + true + + + + docker-buildx-bake-print + + exec + + package + + /usr/bin/env + + sh + -c + + 2>&1 \ + docker buildx bake \ + --print \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl + + + + + + docker-buildx-bake + + exec + + package + + /usr/bin/env + + sh + -c + + 2>&1 \ + docker buildx bake \ + --progress plain \ + --pull \ + --load \ + --set *.platform=${container_image.platforms} \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl + + + + + + + + + + + + amd64 + + + x86_64 + + + + linux/amd64 + + + + arm64 + + + aarch64 + + + + linux/arm64 + + + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/Dockerfile b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/Dockerfile new file mode 100644 index 0000000000..847724c09e --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/Dockerfile @@ -0,0 +1,161 @@ +# syntax=docker/dockerfile:1.4 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# hadolint global ignore=DL3020 + +ARG IMG_BASE='golang' +ARG IMG_BASE_TAG='1.19-alpine' +ARG BASE_IMG_ALPINE='alpine' +ARG BASE_IMG_TAG_ALPINE='latest' +ARG YQ_IMG='mikefarah/yq' +ARG YQ_IMG_TAG='latest' + +FROM ${BASE_IMG_ALPINE}:${BASE_IMG_TAG_ALPINE} as alpine +RUN apk add --no-cache bash~=5 + +FROM ${YQ_IMG}:${YQ_IMG_TAG} as yq + +## -- Stages kubectl_${TARGETARCH} -- +# Define stages that facilitate bringing in platform-specific binaries. +FROM alpine as kubectl_amd64 +ARG KUBECTL_SHA_AMD64_URL +ARG KUBECTL_SHA_AMD64 +ARG KUBECTL_BIN_AMD64_URL +ARG KUBECTL_BIN_AMD64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUBECTL_SHA_AMD64_URL} /tmp/ +ADD --link ${KUBECTL_BIN_AMD64_URL} /tmp/ +RUN echo "$(cat "${KUBECTL_SHA_AMD64}") ${KUBECTL_BIN_AMD64}" | sha512sum -c +ENV KUBECTL_BIN "${KUBECTL_BIN_AMD64}" + +FROM alpine as kubectl_arm64 +ARG KUBECTL_SHA_ARM64_URL +ARG KUBECTL_SHA_ARM64 +ARG KUBECTL_BIN_ARM64_URL +ARG KUBECTL_BIN_ARM64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUBECTL_SHA_ARM64_URL} /tmp/ +ADD --link ${KUBECTL_BIN_ARM64_URL} /tmp/ +RUN echo "$(cat "${KUBECTL_SHA_ARM64}") ${KUBECTL_BIN_ARM64}" | sha512sum -c +ENV KUBECTL_BIN "${KUBECTL_BIN_ARM64}" + +ARG TARGETARCH +# hadolint ignore=DL3006 +FROM kubectl_${TARGETARCH} as kubectl +RUN mv "/tmp/${KUBECTL_BIN}" /tmp/kubectl \ + && chmod a+x /tmp/kubectl + +## -- Stages kuttl_${TARGETARCH} -- +# Define stages that facilitate bringing in platform-specific binaries. +FROM alpine as kuttl_amd64 +ARG KUTTL_CHECKSUMS_URL +ARG KUTTL_CHECKSUMS +ARG KUTTL_BIN_AMD64_URL +ARG KUTTL_BIN_AMD64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUTTL_CHECKSUMS_URL} /tmp/ +ADD --link ${KUTTL_BIN_AMD64_URL} /tmp/ +RUN sha256sum -c <(grep "${KUTTL_BIN_AMD64}" "${KUTTL_CHECKSUMS}") +ENV KUTTL_BIN "${KUTTL_BIN_AMD64}" + +FROM alpine as kuttl_arm64 +ARG KUTTL_CHECKSUMS_URL +ARG KUTTL_CHECKSUMS +ARG KUTTL_BIN_ARM64_URL +ARG KUTTL_BIN_ARM64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUTTL_CHECKSUMS_URL} /tmp/ +ADD --link ${KUTTL_BIN_ARM64_URL} /tmp/ +RUN sha256sum -c <(grep "${KUTTL_BIN_ARM64}" "${KUTTL_CHECKSUMS}") +ENV KUTTL_BIN "${KUTTL_BIN_ARM64}" + +ARG TARGETARCH +# hadolint ignore=DL3006 +FROM kuttl_${TARGETARCH} as kuttl +RUN mv "/tmp/${KUTTL_BIN}" /tmp/kubectl-kuttl \ + && chmod a+x /tmp/kubectl-kuttl + +## -- Stages kustomize_${TARGETARCH} -- +# Define stages that facilitate bringing in platform-specific binaries. +FROM alpine as kustomize_amd64 +ARG KUSTOMIZE_CHECKSUMS_URL +ARG KUSTOMIZE_CHECKSUMS +ARG KUSTOMIZE_BIN_AMD64_TGZ_URL +ARG KUSTOMIZE_BIN_AMD64_TGZ +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUSTOMIZE_CHECKSUMS_URL} /tmp/ +ADD --link ${KUSTOMIZE_BIN_AMD64_TGZ_URL} /tmp/ +RUN sha256sum -c <(grep "${KUSTOMIZE_BIN_AMD64_TGZ}" "${KUSTOMIZE_CHECKSUMS}") +ENV KUSTOMIZE_BIN_TGZ "${KUSTOMIZE_BIN_AMD64_TGZ}" + +FROM alpine as kustomize_arm64 +ARG KUSTOMIZE_CHECKSUMS_URL +ARG KUSTOMIZE_CHECKSUMS +ARG KUSTOMIZE_BIN_ARM64_TGZ_URL +ARG KUSTOMIZE_BIN_ARM64_TGZ +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUSTOMIZE_CHECKSUMS_URL} /tmp/ +ADD --link ${KUSTOMIZE_BIN_ARM64_TGZ_URL} /tmp/ +RUN sha256sum -c <(grep "${KUSTOMIZE_BIN_ARM64_TGZ}" "${KUSTOMIZE_CHECKSUMS}") +ENV KUSTOMIZE_BIN_TGZ "${KUSTOMIZE_BIN_ARM64_TGZ}" + +ARG TARGETARCH +# hadolint ignore=DL3006 +FROM kustomize_${TARGETARCH} as kustomize +RUN tar xzf "/tmp/${KUSTOMIZE_BIN_TGZ}" \ + && chmod a+x /tmp/kustomize + +FROM ${IMG_BASE}:${IMG_BASE_TAG} as final +ARG IMG_BASE +ARG IMG_BASE_TAG + +COPY --from=yq /usr/bin/yq /usr/bin/yq +COPY --from=kubectl /tmp/kubectl /usr/local/bin/ +COPY --from=kuttl /tmp/kubectl-kuttl /usr/local/bin/ +COPY --from=kustomize /tmp/kustomize /usr/local/bin/ +COPY src/main/docker/entrypoint.sh /bin/ + +# nonroot user as defined in https://github.com/GoogleContainerTools/distroless +ENV NON_ROOT_USER=nonroot +ENV NON_ROOT_USER_ID=65532 +ENV NON_ROOT_USER_HOME=/home/nonroot + +# hadolint ignore=DL3018 +RUN apk add --update --no-cache \ + bash~=5 \ + docker-cli \ + && adduser -D -u "${NON_ROOT_USER_ID}" -h "${NON_ROOT_USER_HOME}" "${NON_ROOT_USER}" + +# replicate the test-related bits generated by `kubebuilder` into its Makefile. +ENV GOBIN="${GOPATH}/bin" +ENV ENVTEST_K8S_VERSION='1.23.x' +RUN chmod a+x /bin/entrypoint.sh \ + && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +WORKDIR "${NON_ROOT_USER_HOME}" +USER "${NON_ROOT_USER}" +RUN "${GOBIN}/setup-envtest" use "${ENVTEST_K8S_VERSION}" +# disable downloading remote content henceforth +ENV ENVTEST_INSTALLED_ONLY=true + +ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.hcl b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.hcl new file mode 100644 index 0000000000..54c4d83017 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.hcl @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A convenience script for build the kuttl image. +# See hbase-kubernetes-deployment/dockerfiles/kuttl/README.md +# + +# input variables +variable KUBECTL_SHA_AMD64_URL {} +variable KUBECTL_SHA_AMD64 {} +variable KUBECTL_BIN_AMD64_URL {} +variable KUBECTL_BIN_AMD64 {} +variable KUBECTL_SHA_ARM64_URL {} +variable KUBECTL_SHA_ARM64 {} +variable KUBECTL_BIN_ARM64_URL {} +variable KUBECTL_BIN_ARM64 {} +variable KUTTL_CHECKSUMS_URL {} +variable KUTTL_CHECKSUMS {} +variable KUTTL_BIN_AMD64_URL {} +variable KUTTL_BIN_AMD64 {} +variable KUTTL_BIN_ARM64_URL {} +variable KUTTL_BIN_ARM64 {} +variable KUSTOMIZE_CHECKSUMS_URL {} +variable KUSTOMIZE_CHECKSUMS {} +variable KUSTOMIZE_BIN_AMD64_TGZ_URL {} +variable KUSTOMIZE_BIN_AMD64_TGZ {} +variable KUSTOMIZE_BIN_ARM64_TGZ_URL {} +variable KUSTOMIZE_BIN_ARM64_TGZ {} + +# output variables +variable USER { + default = "apache" +} +variable IMAGE_TAG { + default = "latest" +} +variable IMAGE_NAME { + default = "${USER}/hbase/operator-tools/kuttl" +} + +group default { + targets = [ "kuttl" ] +} + +target kuttl { + dockerfile = "src/main/docker/Dockerfile" + args = { + KUBECTL_SHA_AMD64_URL = KUBECTL_SHA_AMD64_URL + KUBECTL_SHA_AMD64 = KUBECTL_SHA_AMD64 + KUBECTL_BIN_AMD64_URL = KUBECTL_BIN_AMD64_URL + KUBECTL_BIN_AMD64 = KUBECTL_BIN_AMD64 + KUBECTL_SHA_ARM64_URL = KUBECTL_SHA_ARM64_URL + KUBECTL_SHA_ARM64 = KUBECTL_SHA_ARM64 + KUBECTL_BIN_ARM64_URL = KUBECTL_BIN_ARM64_URL + KUBECTL_BIN_ARM64 = KUBECTL_BIN_ARM64 + KUTTL_CHECKSUMS_URL = KUTTL_CHECKSUMS_URL + KUTTL_CHECKSUMS = KUTTL_CHECKSUMS + KUTTL_BIN_AMD64_URL = KUTTL_BIN_AMD64_URL + KUTTL_BIN_AMD64 = KUTTL_BIN_AMD64 + KUTTL_BIN_ARM64_URL = KUTTL_BIN_ARM64_URL + KUTTL_BIN_ARM64 = KUTTL_BIN_ARM64 + KUSTOMIZE_CHECKSUMS_URL = KUSTOMIZE_CHECKSUMS_URL + KUSTOMIZE_CHECKSUMS = KUSTOMIZE_CHECKSUMS + KUSTOMIZE_BIN_AMD64_TGZ_URL = KUSTOMIZE_BIN_AMD64_TGZ_URL + KUSTOMIZE_BIN_AMD64_TGZ = KUSTOMIZE_BIN_AMD64_TGZ + KUSTOMIZE_BIN_ARM64_TGZ_URL = KUSTOMIZE_BIN_ARM64_TGZ_URL + KUSTOMIZE_BIN_ARM64_TGZ = KUSTOMIZE_BIN_ARM64_TGZ + } + target = "final" + platforms = [ + "linux/amd64", + "linux/arm64" + ] + tags = [ "${IMAGE_NAME}:${IMAGE_TAG}" ] +} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.override.hcl b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.override.hcl new file mode 100644 index 0000000000..fa33354830 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.override.hcl @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Externalize default values of build parameters and document how to retrieve them. +# + +function "basename" { + params = [a] + result = split("/", a)[length(split("/", a)) - 1] +} + +variable KUBECTL_VERSION { + default = "1.24.10" +} + +variable KUBECTL_SHA_AMD64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha512" +} + +variable KUBECTL_SHA_AMD64 { + default = "${basename(KUBECTL_SHA_AMD64_URL)}" +} + +variable KUBECTL_BIN_AMD64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" +} + +variable KUBECTL_BIN_AMD64 { + default = "${basename(KUBECTL_BIN_AMD64_URL)}" +} + +variable KUBECTL_SHA_ARM64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/arm64/kubectl.sha512" +} + +variable KUBECTL_SHA_ARM64 { + default = "${basename(KUBECTL_SHA_ARM64_URL)}" +} + +variable KUBECTL_BIN_ARM64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/arm64/kubectl" +} + +variable KUBECTL_BIN_ARM64 { + default = "${basename(KUBECTL_BIN_ARM64_URL)}" +} + +variable KUTTL_VERSION { + default = "0.15.0" +} + +variable KUTTL_CHECKSUMS_URL { + default = "https://github.com/kudobuilder/kuttl/releases/download/v${KUTTL_VERSION}/checksums.txt" +} + +variable KUTTL_CHECKSUMS { + default = "${basename(KUTTL_CHECKSUMS_URL)}" +} + +variable KUTTL_BIN_AMD64_URL { + default = "https://github.com/kudobuilder/kuttl/releases/download/v${KUTTL_VERSION}/kubectl-kuttl_${KUTTL_VERSION}_linux_x86_64" +} + +variable KUTTL_BIN_AMD64 { + default = "${basename(KUTTL_BIN_AMD64_URL)}" +} + +variable KUTTL_BIN_ARM64_URL { + default = "https://github.com/kudobuilder/kuttl/releases/download/v${KUTTL_VERSION}/kubectl-kuttl_${KUTTL_VERSION}_linux_arm64" +} + +variable KUTTL_BIN_ARM64 { + default = "${basename(KUTTL_BIN_ARM64_URL)}" +} + +variable KUSTOMIZE_VERSION { + default = "4.5.4" +} + +variable KUSTOMIZE_CHECKSUMS_URL { + default = "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/checksums.txt" +} + +variable KUSTOMIZE_CHECKSUMS { + default = "${basename(KUSTOMIZE_CHECKSUMS_URL)}" +} + +variable KUSTOMIZE_BIN_AMD64_TGZ_URL { + default = "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" +} + +variable KUSTOMIZE_BIN_AMD64_TGZ { + default = "${basename(KUSTOMIZE_BIN_AMD64_TGZ_URL)}" +} + +variable KUSTOMIZE_BIN_ARM64_TGZ_URL { + default = "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_arm64.tar.gz" +} + +variable KUSTOMIZE_BIN_ARM64_TGZ { + default = "${basename(KUSTOMIZE_BIN_ARM64_TGZ_URL)}" +} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/entrypoint.sh b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/entrypoint.sh new file mode 100644 index 0000000000..8436e82600 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/entrypoint.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -o pipefail +set -x + +declare ENVTEST_K8S_VERSION + +declare KUBEBUILDER_ASSETS +KUBEBUILDER_ASSETS="$(setup-envtest use -i "${ENVTEST_K8S_VERSION}" -p path)" +export KUBEBUILDER_ASSETS + +/usr/local/bin/kubectl kuttl test "$@" diff --git a/hbase-kubernetes-deployment/pom.xml b/hbase-kubernetes-deployment/pom.xml new file mode 100644 index 0000000000..2bfe45d012 --- /dev/null +++ b/hbase-kubernetes-deployment/pom.xml @@ -0,0 +1,47 @@ + + + + 4.0.0 + + hbase-operator-tools + org.apache.hbase.operator.tools + ${revision} + .. + + + hbase-kubernetes-deployment + Apache HBase - Kubernetes Deployment + Resource definitions for deploying HBase on Kubernetes. + pom + + + hbase-kubernetes-hadoop-image + hbase-kubernetes-kustomize + hbase-kubernetes-testing-image + + + + + + + diff --git a/pom.xml b/pom.xml index bd3637ad18..392d4e242a 100644 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,7 @@ hbase-table-reporter hbase-hbck2 + hbase-kubernetes-deployment hbase-operator-tools-assembly @@ -162,6 +163,12 @@ hbase-tools ${project.version} + + org.apache.hbase.operator.tools + hbase-kubernetes-testing-image + pom + ${project.version} + @@ -321,6 +328,11 @@ true + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + @@ -428,6 +440,13 @@ + + + kr.motd.maven + os-maven-plugin + 1.7.1 + +