1 # A headless service to create DNS records.
5 name: {{ template "hdfs-k8s.namenode.fullname" . }}
7 app: {{ template "hdfs-k8s.namenode.name" . }}
8 chart: {{ template "hdfs-k8s.subchart" . }}
9 release: {{ .Release.Name }}
11 # TODO: Deprecated. Replace tolerate-unready-endpoints with
12 # v1.Service.PublishNotReadyAddresses.
13 service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
22 app: {{ template "hdfs-k8s.namenode.name" . }}
23 release: {{ .Release.Name }}
25 apiVersion: policy/v1beta1
26 kind: PodDisruptionBudget
28 name: {{ template "hdfs-k8s.namenode.fullname" . }}
30 app: {{ template "hdfs-k8s.namenode.name" . }}
31 chart: {{ template "hdfs-k8s.subchart" . }}
32 release: {{ .Release.Name }}
36 app: {{ template "hdfs-k8s.namenode.name" . }}
37 release: {{ .Release.Name }}
40 # Provides namenode helper scripts. Most of them are start scripts
41 # that meet different needs.
42 # TODO: Support upgrade of metadata in case a new Hadoop version requires it.
46 name: {{ template "hdfs-k8s.namenode.fullname" . }}-scripts
48 app: {{ template "hdfs-k8s.namenode.name" . }}
49 chart: {{ template "hdfs-k8s.subchart" . }}
50 release: {{ .Release.Name }}
52 # A bootstrap script which will start namenode daemons after conducting
53 # optional metadata initialization steps. The metadata initialization
54 # steps will take place in case the metadata dir is empty,
55 # which will be the case only for the very first run. The specific steps
56 # will differ depending on whether the namenode is active or standby.
57 # We also assume, for the very first run, namenode-0 will be active and
58 # namenode-1 will be standby as StatefulSet will launch namenode-0 first
59 # and zookeeper will determine the sole namenode to be the active one.
60 # For active namenode, the initialization steps will format the metadata,
61 # zookeeper dir and journal node data entries.
62 # For standby namenode, the initialization steps will simply receieve
63 # the first batch of metadata updates from the journal node.
66 # Exit on error. Append "|| true" if you expect an error.
68 # Exit on error inside any functions or subshells.
70 # Do not allow use of undefined vars. Use ${VAR:-} to use an undefined VAR
72 # Catch an error in command pipes. e.g. mysqldump fails (but gzip succeeds)
73 # in `mysqldump |gzip`
75 # Turn on traces, useful while debugging.
78 _HDFS_BIN=$HADOOP_PREFIX/bin/hdfs
79 _METADATA_DIR=/hadoop/dfs/name/current
80 if [[ "$MY_POD" = "$NAMENODE_POD_0" ]]; then
81 if [[ ! -d $_METADATA_DIR ]]; then
82 $_HDFS_BIN --config $HADOOP_CONF_DIR namenode -format \
83 -nonInteractive hdfs-k8s ||
84 (rm -rf $_METADATA_DIR; exit 1)
86 _ZKFC_FORMATTED=/hadoop/dfs/name/current/.hdfs-k8s-zkfc-formatted
87 if [[ ! -f $_ZKFC_FORMATTED ]]; then
88 _OUT=$($_HDFS_BIN --config $HADOOP_CONF_DIR zkfc -formatZK -nonInteractive 2>&1)
89 # zkfc masks fatal exceptions and returns exit code 0
90 (echo $_OUT | grep -q "FATAL") && exit 1
91 touch $_ZKFC_FORMATTED
93 elif [[ "$MY_POD" = "$NAMENODE_POD_1" ]]; then
94 if [[ ! -d $_METADATA_DIR ]]; then
95 $_HDFS_BIN --config $HADOOP_CONF_DIR namenode -bootstrapStandby \
97 (rm -rf $_METADATA_DIR; exit 1)
100 $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR start zkfc
101 $_HDFS_BIN --config $HADOOP_CONF_DIR namenode
103 # A start script that will just hang indefinitely. A user can then get
104 # inside the pod and debug. Or a user can conduct a custom manual operations.
107 tail -f /var/log/dmesg
109 # A start script that has user specified content. Can be used to conduct
110 # ad-hoc operation as specified by a user.
111 custom-run.sh: {{ .Values.customRunScript | quote }}
113 apiVersion: apps/v1beta1
116 name: {{ template "hdfs-k8s.namenode.fullname" . }}
118 app: {{ template "hdfs-k8s.namenode.name" . }}
119 chart: {{ template "hdfs-k8s.subchart" . }}
120 release: {{ .Release.Name }}
122 serviceName: {{ template "hdfs-k8s.namenode.fullname" . }}
127 app: {{ template "hdfs-k8s.namenode.name" . }}
128 release: {{ .Release.Name }}
129 {{- if .Values.podAnnotations }}
131 {{ toYaml .Values.podAnnotations | indent 8 }}
134 {{- if .Values.hostNetworkEnabled }}
135 # Use hostNetwork so datanodes connect to namenode without going through an overlay network
136 # like weave. Otherwise, namenode fails to see physical IP address of datanodes.
137 # Disabling this will break data locality as namenode will see pod virtual IPs and fails to
138 # equate them with cluster node physical IPs associated with data nodes.
139 # We currently disable this only for CI on minikube.
142 dnsPolicy: ClusterFirstWithHostNet
144 dnsPolicy: ClusterFirst
146 {{- if .Values.affinity }}
148 {{ toYaml .Values.affinity | indent 8 }}
149 {{- else if .Values.global.defaultAffinityEnabled }}
152 requiredDuringSchedulingIgnoredDuringExecution:
158 - {{ template "hdfs-k8s.namenode.name" . }}
162 - {{ .Release.Name }}
163 topologyKey: "kubernetes.io/hostname"
165 {{- if .Values.nodeSelector }}
167 {{ toYaml .Values.nodeSelector | indent 8 }}
169 {{- if .Values.tolerations }}
171 {{ toYaml .Values.tolerations | indent 8 }}
174 # TODO: Support hadoop version as option.
175 - name: hdfs-namenode
176 image: uhopper/hadoop-namenode:2.7.2
178 - name: HADOOP_CUSTOM_CONF_DIR
179 value: /etc/hadoop-custom-conf
180 - name: MULTIHOMED_NETWORK
182 # Used by the start script below.
186 fieldPath: metadata.name
187 - name: NAMENODE_POD_0
188 value: {{ template "namenode-pod-0" . }}
189 - name: NAMENODE_POD_1
190 value: {{ template "namenode-pod-1" . }}
191 command: ['/bin/sh', '-c']
192 # The start script is provided by a config map.
194 - /entrypoint.sh "/nn-scripts/{{ .Values.namenodeStartScript }}"
196 - containerPort: 8020
198 - containerPort: 50070
202 mountPath: /nn-scripts
204 # Mount a subpath of the volume so that the name subdir would be a
205 # brand new empty dir. This way, we won't get affected by existing
206 # files in the volume top dir.
208 mountPath: /hadoop/dfs/name
211 mountPath: /etc/hadoop-custom-conf
213 {{- if .Values.global.kerberosEnabled }}
214 - name: kerberos-config
215 mountPath: /etc/krb5.conf
216 subPath: {{ .Values.global.kerberosConfigFileName }}
218 - name: kerberos-keytab-copy
219 mountPath: /etc/security/
222 {{- if .Values.global.kerberosEnabled }}
224 - name: copy-kerberos-keytab
225 image: busybox:1.27.1
226 command: ['sh', '-c']
228 - cp /kerberos-keytabs/${MY_KERBEROS_NAME}*.keytab /kerberos-keytab-copy/hdfs.keytab
230 - name: MY_KERBEROS_NAME
233 {{- if .Values.hostNetworkEnabled }}
234 fieldPath: spec.nodeName
236 fieldPath: metadata.name
239 - name: kerberos-keytabs
240 mountPath: /kerberos-keytabs
241 - name: kerberos-keytab-copy
242 mountPath: /kerberos-keytab-copy
244 restartPolicy: Always
248 name: {{ template "hdfs-k8s.namenode.fullname" . }}-scripts
252 name: {{ template "hdfs-k8s.config.fullname" . }}
253 {{- if .Values.global.kerberosEnabled }}
254 - name: kerberos-config
256 name: {{ template "krb5-configmap" . }}
257 - name: kerberos-keytabs
259 secretName: {{ template "krb5-keytabs-secret" . }}
260 - name: kerberos-keytab-copy
263 {{- if .Values.global.podSecurityContext.enabled }}
265 runAsUser: {{ .Values.global.podSecurityContext.runAsUser }}
266 fsGroup: {{ .Values.global.podSecurityContext.fsGroup }}
268 volumeClaimTemplates:
273 - {{ .Values.persistence.accessMode | quote }}
276 storage: {{ .Values.persistence.size | quote }}
277 {{- if .Values.persistence.storageClass }}
278 {{- if (eq "-" .Values.persistence.storageClass) }}
281 storageClassName: "{{ .Values.persistence.storageClass }}"
284 {{- if .Values.persistence.selector }}
286 {{ toYaml .Values.persistence.selector | indent 10 }}