I have an Elasticsearch DB running on Kubernetes exposed to my_domain.com/elastic as an Istio virtual service, which I have no problem accessing via the browser (as in I get to login successfully to the endpoint). I can also query the DB with Python's Requests. But I can't access the DB with the official python client if I use my_domain.com/elastic. The LoadBalancer IP works perfectly well even with the client. What am I missing? I have SSL certificates set up for my_domain.com via Cert-Manager and CloudFlare.

This works:

import requests
import os
data = ' { "query": { "match_all": {} } }'
headers = {'Content-Type': 'application/json'}
auth= ('elastic', os.environ['ELASTIC_PASSWORD']) 
response = requests.post('https://mydomain.cloud/elastic/_search', auth=auth,  data=data, headers=headers)
print(response.text)

This doesn't work (I have tried a number of different parameters):

from datetime import datetime
import os
from elasticsearch import Elasticsearch, RequestsHttpConnection


es = Elasticsearch(,
   [{'host': 'mydomain.cloud', 'port': 443, 'url_prefix': 'elastic', 'use_ssl': True}],
    http_auth=('elastic', os.environ['ELASTIC_PASSWORD']), # 1Password or kubectl get secret elastic-cluster-es-elastic-user -o go-template='{{.data.elastic | base64decode}}' -n elastic-system
    schema='https'#, verify_certs=False,
    # use_ssl=True,
    # connection_class = RequestsHttpConnection,
    # port=443,
)

# if not es.ping():
#     raise ValueError("Connection failed")

doc = {
    'author': 'kimchy',
    'text': 'Elasticsearch: cool. bonsai cool.',
    'timestamp': datetime.now(),
}
res = es.index(index="test-index", id=1, document=doc)
print(res['result'])

res = es.get(index="test-index", id=1)
print(res['_source'])

es.indices.refresh(index="test-index")

res = es.search(index="test-index", query={"match_all": {}})
print("Got %d Hits:" % res['hits']['total']['value'])
for hit in res['hits']['hits']:
    print("%(timestamp)s %(author)s: %(text)s" % hit["_source"])

The resulting error:

elasticsearch.exceptions.RequestError: RequestError(400, 'no handler found for uri [//test-index/_doc/1] and method [PUT]', 'no handler found for uri [//test-index/_doc/1] and method [PUT]') 

cluster.yaml

apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: elastic-cluster
  namespace: elastic-system
spec:
  version: 7.15.2
  http:
    # tls:
    #   selfSignedCertificate:
    #      disabled: true
    service:
      spec:
        type: LoadBalancer
  nodeSets:
  - name: master-nodes
    count: 2
    config:
      node.roles: ["master"]
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data # Do not change this name unless you set up a volume mount for the data path.
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 5Gi
        storageClassName: local-storage
  - name: data-nodes
    count: 2
    config:
      node.roles: ["data"]
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data # Do not change this name unless you set up a volume mount for the data path.
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 10Gi
        storageClassName: local-storage
    podTemplate:
      # metadata:
      #   annotations:
      #     traffic.sidecar.istio.io/includeInboundPorts: "*"
      #     traffic.sidecar.istio.io/excludeOutboundPorts: "9300" 
      #     traffic.sidecar.istio.io/excludeInboundPorts: "9300"
      spec:
        # automountServiceAccountToken: true
        containers:
        - name: elasticsearch
          resources:
            requests:
              memory: 4Gi
              cpu: 3
            limits:
              memory: 4Gi
              cpu: 3

virtual-service.yaml

apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: elastic-vts
  namespace: elastic-system
spec:
  hosts:
  - "mydomain.cloud"
  gateways:
  - istio-system/gateway
  http:
  - match:
    - port: 443
    - uri:
        prefix: /elastic
    rewrite:
      uri: /
    route:
    - destination:
        host: elastic-cluster-es-http.elastic-system.svc.cluster.local
        port:
          number: 9200

destination-rule.yaml

apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  name: elastic-destination-rule
  namespace: elastic-system
spec:
  host: elastic-cluster-es-http.elastic-system.svc.cluster.local
  trafficPolicy:
    tls:
      mode: SIMPLE

gateway.yaml

apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: gateway
  namespace: istio-system
spec:
  selector:
    istio: ingressgateway
  servers:
  - port:
      number: 80
      name: http
      protocol: HTTP
    hosts:
    - 'mydomain.cloud'
    tls:
      httpsRedirect: true
  - port:
      number: 443
      name: https
      protocol: HTTPS
    hosts:
      - 'mydomain.cloud'
    tls:
      mode: SIMPLE
      credentialName: letsencrypt-staging-tls