I am very excited by using Elasticsearch on Kubernetes with Spark but I cant' authenticate my job to it. I successfully installed ECK (without TLS => for test) on my GKE cluster with the following template (thanks to Helm):

 apiVersion: v1
 kind: Secret
 metadata:
   name: smart-agriculture-elasticsearch-es-elastic-user # I override the default password for user "elastic" created by ECK during its installation
   namespace: dev
 type: Opaque
 data:
   elastic: dG90bwo= # password is "toto" in base64
 ---
 apiVersion: elasticsearch.k8s.elastic.co/v1
 kind: Elasticsearch
 metadata:
   name: smart-agriculture-elasticsearch
   namespace: dev
 spec:
   version: 7.6.1
   nodeSets:
     - name: default
       count: 1
       config:
         node.master: true
         node.data: true
         node.ingest: true
         node.store.allow_mmap: false
   http:
     tls:
       selfSignedCertificate:
         disabled: true

Then I want to use my spark job to connect to my Elasticsearch cluster, here is my elasticsearch dependency and my hello world code in scala

--

<dependency>
  <groupId>org.elasticsearch</groupId>
  <artifactId>elasticsearch-spark-20_2.11</artifactId>
  <version>7.6.1</version>
</dependency>   

--

import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import org.elasticsearch.spark._

object ElasticSparkHelloWorld {
  def main(args: Array[String]) {

   val conf = new SparkConf().setAppName("spark-es-to-parquet").setMaster("k8s://https://10.0.0.1:443")
   conf.set("executor.instances","2")
   conf.set("kubernetes.namespace", "dev")
   conf.set("kubernetes.authenticate.driver.serviceAccountName", "spark-sa") # searvice account created in another template, it works !
   conf.set("es.index.auto.create", "false")
   conf.set("es.nodes.wan.only", "true")
   conf.set("es.nodes", "http://smart-agriculture-elasticsearch-es-http")
   conf.set("es.port", "9200")
   conf.set("es.net.http.auth.user", "elastic") # user
   conf.set("es.net.http.auth.pass", "toto") # password

   val sc = new SparkContext(conf)

   val numbers = Map("one" -> 1, "two" -> 2, "three" -> 3)
   val airports = Map("arrival" -> "Otopeni", "SFO" -> "San Fran")

   sc.makeRDD(
     Seq(numbers, airports)
   ).saveToEs("spark/docs")
 }
}

However, I get the following error in spark that I don't really understand:

Caused by: org.elasticsearch.hadoop.rest.EsHadoopInvalidRequest: org.elasticsearch.hadoop.rest.EsHadoopRemoteException: security_exception: unable to authenticate user [elastic] for REST request [/]

In Elasticsearch logs, I have:

Authentication to realm file1 failed - Password authentication failed for elastic

Does anyone know how to solve this or have information (links, doc..) that I can use ?

Thanks !