vnfs/DAaaS/training-core/charts/kubernetes-HDFS/charts/hdfs-config-k8s/templates/configmap.yaml

   1 apiVersion: v1
   2 kind: ConfigMap
   3 metadata:
   4   name: {{ template "hdfs-k8s.config.fullname" . }}
   5   labels:
   6     app: {{ template "hdfs-k8s.client.name" . }}
   7     chart: {{ template "hdfs-k8s.subchart" . }}
   8     release: {{ .Release.Name }}
   9 data:
  10   core-site.xml: |
  11     <?xml version="1.0"?>
  12     <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
  13     <configuration>
  14     {{- if .Values.global.kerberosEnabled }}
  15       <property>
  16         <name>hadoop.security.authentication</name>
  17         <value>kerberos</value>
  18       </property>
  19       <!--
  20       This is service level RPC authorization, which is separate from HDFS file
  21       level ACLs.  This concerns who can talk to HDFS daemons including
  22       datanodes talking to namenode.  As part of the authorization, namenode
  23       tries to validate that DNS can uniquely traslate the datanode IP to the
  24       hostname in the datanode Kerberos principal.  (i.e. The client IP is what
  25       Kerberos has authenticated). This does not work well when both namenode
  26       and datanodes are using the Kubernetes HostNetwork and namenode is using
  27       the StatefulSet. The same cluster node IP can be mapped to two different
  28       DNS names. So we disable this. Again this is only service level RPC
  29       authorization and does not affect HDFS file level permission ACLs.
  30       -->
  31       <property>
  32         <name>hadoop.security.authorization</name>
  33         <value>false</value>
  34       </property>
  35       <property>
  36         <name>hadoop.rpc.protection</name>
  37         <value>privacy</value>
  38       </property>
  39       <property>
  40         <name>hadoop.user.group.static.mapping.overrides</name>
  41         <value>hdfs=root;</value>
  42       </property>
  43     {{- end }}
  44     {{- range $key, $value := .Values.customHadoopConfig.coreSite }}
  45       <property>
  46         <name>{{ $key }}</name>
  47         <value>{{ $value }}</value>
  48       </property>
  49     {{- end }}
  50     {{- if .Values.global.namenodeHAEnabled }}
  51       <property>
  52         <name>fs.defaultFS</name>
  53         <value>hdfs://hdfs-k8s</value>
  54       </property>
  55       <property>
  56         <name>ha.zookeeper.quorum</name>
  57         <value>{{ template "zookeeper-quorum" . }}</value>
  58       </property>
  59     {{- else }}
  60       <property>
  61         <name>fs.defaultFS</name>
  62         <value>hdfs://{{ template "namenode-svc-0" . }}:8020</value>
  63       </property>
  64     {{- end }}
  65     </configuration>
  66   hdfs-site.xml: |
  67     <?xml version="1.0"?>
  68     <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
  69     <configuration>
  70     {{- if .Values.global.kerberosEnabled }}
  71       <property>
  72         <name>dfs.block.access.token.enable</name>
  73         <value>true</value>
  74       </property>
  75       <property>
  76         <name>dfs.encrypt.data.transfer</name>
  77         <value>true</value>
  78       </property>
  79       <property>
  80         <name>dfs.namenode.kerberos.principal</name>
  81         <value>{{ template "hdfs-principal" . }}</value>
  82       </property>
  83       {{/*
  84       TODO: Check if the https principal is no longer needed in newer Hadoop version.
  85       */}}
  86       <property>
  87         <name>dfs.namenode.kerberos.https.principal</name>
  88         <value>{{ template "http-principal" . }}</value>
  89       </property>
  90       <property>
  91         <name>dfs.web.authentication.kerberos.principal</name>
  92         <value>{{ template "http-principal" . }}</value>
  93       </property>
  94       <property>
  95         <name>dfs.namenode.keytab.file</name>
  96         <value>/etc/security/hdfs.keytab</value>
  97       </property>
  98       <property>
  99         <name>dfs.journalnode.kerberos.principal</name>
 100         <value>{{ template "hdfs-principal" . }}</value>
 101       </property>
 102       <property>
 103         <name>dfs.journalnode.kerberos.internal.spnego.principal</name>
 104         <value>{{ template "http-principal" . }}</value>
 105       </property>
 106       <property>
 107         <name>dfs.journalnode.keytab.file</name>
 108         <value>/etc/security/hdfs.keytab</value>
 109       </property>
 110       <property>
 111         <name>dfs.datanode.kerberos.principal</name>
 112         <value>{{ template "hdfs-principal" . }}</value>
 113       </property>
 114       <property>
 115         <name>dfs.datanode.kerberos.https.principal</name>
 116         <value>{{ template "http-principal" . }}</value>
 117       </property>
 118       <property>
 119         <name>dfs.datanode.keytab.file</name>
 120         <value>/etc/security/hdfs.keytab</value>
 121       </property>
 122       {{- if .Values.global.jsvcEnabled }}
 123       <property>
 124         <name>dfs.datanode.address</name>
 125         <value>0.0.0.0:1004</value>
 126       </property>
 127       <property>
 128         <name>dfs.datanode.http.address</name>
 129         <value>0.0.0.0:1006</value>
 130       </property>
 131       {{- end }}
 132     {{- end }}
 133     {{- range $key, $value := .Values.customHadoopConfig.hdfsSite }}
 134       <property>
 135         <name>{{ $key }}</name>
 136         <value>{{ $value }}</value>
 137       </property>
 138     {{- end }}
 139     {{- if .Values.global.namenodeHAEnabled }}
 140       <property>
 141         <name>dfs.nameservices</name>
 142         <value>hdfs-k8s</value>
 143       </property>
 144       <property>
 145         <name>dfs.ha.namenodes.hdfs-k8s</name>
 146         <value>nn0,nn1</value>
 147       </property>
 148       <property>
 149         <name>dfs.namenode.rpc-address.hdfs-k8s.nn0</name>
 150         <value>{{ template "namenode-svc-0" . }}:8020</value>
 151       </property>
 152       <property>
 153         <name>dfs.namenode.rpc-address.hdfs-k8s.nn1</name>
 154         <value>{{ template "namenode-svc-1" . }}:8020</value>
 155       </property>
 156       <property>
 157         <name>dfs.namenode.http-address.hdfs-k8s.nn0</name>
 158         <value>{{ template "namenode-svc-0" . }}:50070</value>
 159       </property>
 160       <property>
 161         <name>dfs.namenode.http-address.hdfs-k8s.nn1</name>
 162         <value>{{ template "namenode-svc-1" . }}:50070</value>
 163       </property>
 164       <property>
 165         <name>dfs.namenode.shared.edits.dir</name>
 166         <value>qjournal://{{ template "journalnode-quorum" . }}/hdfs-k8s</value>
 167       </property>
 168       <property>
 169         <name>dfs.ha.automatic-failover.enabled</name>
 170         <value>true</value>
 171       </property>
 172       <property>
 173         <name>dfs.ha.fencing.methods</name>
 174         <value>shell(/bin/true)</value>
 175       </property>
 176       <property>
 177         <name>dfs.journalnode.edits.dir</name>
 178         <value>/hadoop/dfs/journal</value>
 179       </property>
 180       <property>
 181         <name>dfs.client.failover.proxy.provider.hdfs-k8s</name>
 182         <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
 183       </property>
 184     {{- end }}
 185       <property>
 186         <name>dfs.namenode.name.dir</name>
 187         <value>file:///hadoop/dfs/name</value>
 188       </property>
 189       <property>
 190         <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
 191         <value>false</value>
 192       </property>
 193       <property>
 194         <name>dfs.datanode.data.dir</name>
 195         <value>{{ template "datanode-data-dirs" . }}</value>
 196       </property>
 197     </configuration>