Some days ago I created a Kubernetes cluster using Traefik as it's Ingress controller. Afterwards I enabled the Traefik web ui for the subdomain traefik.mydomain.de. Now I'm trying to use Letsencrypt to
This is my full configuration traefik.yml
:
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
rules:
- apiGroups:
- ""
resources:
- services
- endpoints
- secrets
verbs:
- get
- list
- watch
- apiGroups:
- extensions
resources:
- ingresses
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: traefik-ingress-controller
subjects:
- kind: ServiceAccount
name: traefik-ingress-controller
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: traefik-ingress-controller
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: traefik-config
namespace: kube-system
data:
traefik.toml: |
[entryPoints]
[entryPoints.http]
address = ":80"
[entryPoints.http.redirect]
entryPoint = "https"
[entryPoints.https]
address = ":443"
[entryPoints.https.tls]
[acme]
email = "admin@mydomain.de"
storage = "/acme/acme.json"
onHostRule = true
caServer = "https://acme-staging-v02.api.letsencrypt.org/directory"
entryPoint = "https"
[acme.httpChallenge]
entryPoint = "http"
[[acme.domains]]
main = "mydomain.de"
sans = ["traefik.mydomain.de"]
---
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
name: traefik-ingress-controller
namespace: kube-system
labels:
k8s-app: traefik-ingress-lb
spec:
template:
metadata:
labels:
k8s-app: traefik-ingress-lb
name: traefik-ingress-lb
spec:
serviceAccountName: traefik-ingress-controller
terminationGracePeriodSeconds: 60
hostNetwork: true
volumes:
- name: config
configMap:
name: traefik-config
- name: acme
hostPath:
path: /srv/configs/acme.json
type: File
containers:
- image: traefik
name: traefik-ingress-lb
volumeMounts:
- mountPath: "/config"
name: "config"
- mountPath: "/acme/acme.json"
name: "acme"
ports:
- name: http
containerPort: 80
hostPort: 80
- name: https
containerPort: 443
hostPort: 443
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
args:
- --configfile=/config/traefik.toml
- --api
- --kubernetes
- --logLevel=DEBUG
---
kind: Service
apiVersion: v1
metadata:
name: traefik-ingress-service
namespace: kube-system
spec:
selector:
k8s-app: traefik-ingress-lb
ports:
- protocol: TCP
port: 80
name: http
- protocol: TCP
port: 443
name: https
type: NodePort
---
apiVersion: v1
kind: Service
metadata:
name: traefik-web-ui
namespace: kube-system
spec:
selector:
k8s-app: traefik-ingress-lb
ports:
- protocol: TCP
port: 8080
name: webui
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: traefik-web-ui
namespace: kube-system
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: traefik.mydomain.de
http:
paths:
- path: /
backend:
serviceName: traefik-web-ui
servicePort: 8080
The result:
404 page not found
Debug output:
time="2018-05-31T10:54:58Z" level=info msg="Using TOML configuration file /config/traefik.toml"
time="2018-05-31T10:54:58Z" level=info msg="Traefik version v1.6.2 built on 2018-05-22_03:19:06PM"
time="2018-05-31T10:54:58Z" level=info msg="\nStats collection is disabled.\nHelp us improve Traefik by turning this feature on :)\nMore details on: https://docs.traefik.io/basics/#collected-data\n"
time="2018-05-31T10:54:58Z" level=debug msg="Global configuration loaded {\"LifeCycle\":{\"RequestAcceptGraceTimeout\":0,\"GraceTimeOut\":10000000000},\"GraceTimeOut\":0,\"Debug\":false,\"CheckNewVersion\":true,\"SendAnonymousUsage\":false,\"AccessLogsFile\":\"\",\"AccessLog\":null,\"TraefikLogsFile\":\"\",\"TraefikLog\":null,\"Tracing\":null,\"LogLevel\":\"DEBUG\",\"EntryPoints\":{\"http\":{\"Address\":\":80\",\"TLS\":null,\"Redirect\":{\"entryPoint\":\"https\"},\"Auth\":null,\"WhitelistSourceRange\":null,\"WhiteList\":null,\"Compress\":false,\"ProxyProtocol\":null,\"ForwardedHeaders\":{\"Insecure\":true,\"TrustedIPs\":null}},\"https\":{\"Address\":\":443\",\"TLS\":{\"MinVersion\":\"\",\"CipherSuites\":null,\"Certificates\":null,\"ClientCAFiles\":null,\"ClientCA\":{\"Files\":null,\"Optional\":false}},\"Redirect\":null,\"Auth\":null,\"WhitelistSourceRange\":null,\"WhiteList\":null,\"Compress\":false,\"ProxyProtocol\":null,\"ForwardedHeaders\":{\"Insecure\":true,\"TrustedIPs\":null}},\"traefik\":{\"Address\":\":8080\",\"TLS\":null,\"Redirect\":null,\"Auth\":null,\"WhitelistSourceRange\":null,\"WhiteList\":null,\"Compress\":false,\"ProxyProtocol\":null,\"ForwardedHeaders\":{\"Insecure\":true,\"TrustedIPs\":null}}},\"Cluster\":null,\"Constraints\":[],\"ACME\":null,\"DefaultEntryPoints\":[\"http\"],\"ProvidersThrottleDuration\":2000000000,\"MaxIdleConnsPerHost\":200,\"IdleTimeout\":0,\"InsecureSkipVerify\":false,\"RootCAs\":null,\"Retry\":null,\"HealthCheck\":{\"Interval\":30000000000},\"RespondingTimeouts\":null,\"ForwardingTimeouts\":null,\"AllowMinWeightZero\":false,\"Web\":null,\"Docker\":null,\"File\":null,\"Marathon\":null,\"Consul\":null,\"ConsulCatalog\":null,\"Etcd\":null,\"Zookeeper\":null,\"Boltdb\":null,\"Kubernetes\":{\"Watch\":true,\"Filename\":\"\",\"Constraints\":[],\"Trace\":false,\"TemplateVersion\":0,\"DebugLogGeneratedTemplate\":false,\"Endpoint\":\"\",\"Token\":\"\",\"CertAuthFilePath\":\"\",\"DisablePassHostHeaders\":false,\"EnablePassTLSCert\":false,\"Namespaces\":null,\"LabelSelector\":\"\",\"IngressClass\":\"\"},\"Mesos\":null,\"Eureka\":null,\"ECS\":null,\"Rancher\":null,\"DynamoDB\":null,\"ServiceFabric\":null,\"Rest\":null,\"API\":{\"EntryPoint\":\"traefik\",\"Dashboard\":true,\"Debug\":false,\"CurrentConfigurations\":null,\"Statistics\":null},\"Metrics\":null,\"Ping\":null}"
time="2018-05-31T10:54:58Z" level=info msg="Preparing server https &{Address::443 TLS:0xc42057e900 Redirect:<nil> Auth:<nil> WhitelistSourceRange:[] WhiteList:<nil> Compress:false ProxyProtocol:<nil> ForwardedHeaders:0xc420020480} with readTimeout=0s writeTimeout=0s idleTimeout=3m0s"
time="2018-05-31T10:54:59Z" level=info msg="Preparing server http &{Address::80 TLS:<nil> Redirect:0xc420092a80 Auth:<nil> WhitelistSourceRange:[] WhiteList:<nil> Compress:false ProxyProtocol:<nil> ForwardedHeaders:0xc4200204a0} with readTimeout=0s writeTimeout=0s idleTimeout=3m0s"
time="2018-05-31T10:54:59Z" level=info msg="Preparing server traefik &{Address::8080 TLS:<nil> Redirect:<nil> Auth:<nil> WhitelistSourceRange:[] WhiteList:<nil> Compress:false ProxyProtocol:<nil> ForwardedHeaders:0xc4200204c0} with readTimeout=0s writeTimeout=0s idleTimeout=3m0s"
time="2018-05-31T10:54:59Z" level=info msg="Starting provider configuration.providerAggregator {}"
time="2018-05-31T10:54:59Z" level=info msg="Starting server on :443"
time="2018-05-31T10:54:59Z" level=info msg="Starting server on :80"
time="2018-05-31T10:54:59Z" level=info msg="Starting server on :8080"
time="2018-05-31T10:54:59Z" level=info msg="Starting provider *kubernetes.Provider {\"Watch\":true,\"Filename\":\"\",\"Constraints\":[],\"Trace\":false,\"TemplateVersion\":0,\"DebugLogGeneratedTemplate\":false,\"Endpoint\":\"\",\"Token\":\"\",\"CertAuthFilePath\":\"\",\"DisablePassHostHeaders\":false,\"EnablePassTLSCert\":false,\"Namespaces\":null,\"LabelSelector\":\"\",\"IngressClass\":\"\"}"
time="2018-05-31T10:54:59Z" level=info msg="Starting provider *acme.Provider {\"Email\":\"admin@mydomain.de\",\"ACMELogging\":false,\"CAServer\":\"https://acme-staging-v02.api.letsencrypt.org/directory\",\"Storage\":\"/acme/acme.json\",\"EntryPoint\":\"https\",\"OnHostRule\":true,\"OnDemand\":false,\"DNSChallenge\":null,\"HTTPChallenge\":{\"EntryPoint\":\"http\"},\"Domains\":[{\"Main\":\"mydomain.de\",\"SANs\":[\"traefik.mydomain.de\"]}],\"Store\":{}}"
time="2018-05-31T10:54:59Z" level=debug msg="Using Ingress label selector: \"\""
time="2018-05-31T10:54:59Z" level=info msg="ingress label selector is: \"\""
time="2018-05-31T10:54:59Z" level=info msg="Creating in-cluster Provider client"
time="2018-05-31T10:54:59Z" level=info msg="Testing certificate renew..."
time="2018-05-31T10:54:59Z" level=debug msg="Configuration received from provider ACME: {\"tls\":[{\"EntryPoints\":[\"https\"],\"Certificate\":{\"CertFile\":\"-----BEGIN CERTIFICATE-----<<< cert here >>>-----END CERTIFICATE-----\\n\\n-----BEGIN CERTIFICATE-----<<< another cert here >>>-----END CERTIFICATE-----\\n\",\"KeyFile\":\"-----BEGIN RSA PRIVATE<<< rsa data here >>>-----END RSA PRIVATE KEY-----\\n\"}}]}"
time="2018-05-31T10:54:59Z" level=debug msg="Looking for provided certificate(s) to validate [\"mydomain.de\" \"traefik.mydomain.de\"]..."
time="2018-05-31T10:54:59Z" level=debug msg="No ACME certificate to generate for domains [\"mydomain.de\" \"traefik.mydomain.de\"]."
time="2018-05-31T10:54:59Z" level=debug msg="Add certificate for domains mydomain.de,traefik.mydomain.de"
time="2018-05-31T10:54:59Z" level=info msg="Server configuration reloaded on :8080"
time="2018-05-31T10:54:59Z" level=info msg="Server configuration reloaded on :443"
time="2018-05-31T10:54:59Z" level=info msg="Server configuration reloaded on :80"
time="2018-05-31T10:54:59Z" level=debug msg="Received Kubernetes event kind *v1.Service"
time="2018-05-31T10:54:59Z" level=debug msg="Configuration received from provider kubernetes: {\"backends\":{\"traefik.mydomain.de/\":{\"servers\":{\"traefik-ingress-controller-lqkjn\":{\"url\":\"https://11.22.33.44:8080\",\"weight\":1}},\"loadBalancer\":{\"method\":\"wrr\"}}},\"frontends\":{\"traefik.mydomain.de/\":{\"entryPoints\":[\"http\"],\"backend\":\"traefik.mydomain.de/\",\"routes\":{\"/\":{\"rule\":\"PathPrefix:/\"},\"traefik.mydomain.de\":{\"rule\":\"Host:traefik.mydomain.de\"}},\"passHostHeader\":true,\"priority\":0,\"basicAuth\":[]}}}"
time="2018-05-31T10:54:59Z" level=debug msg="Creating frontend traefik.mydomain.de/"
time="2018-05-31T10:54:59Z" level=debug msg="Wiring frontend traefik.mydomain.de/ to entryPoint http"
time="2018-05-31T10:54:59Z" level=debug msg="Creating route traefik.mydomain.de Host:traefik.mydomain.de"
time="2018-05-31T10:54:59Z" level=debug msg="Creating route / PathPrefix:/"
time="2018-05-31T10:54:59Z" level=debug msg="Creating entry point redirect http -> https"
time="2018-05-31T10:54:59Z" level=debug msg="Creating backend traefik.mydomain.de/"
time="2018-05-31T10:54:59Z" level=debug msg="Creating load-balancer wrr"
time="2018-05-31T10:54:59Z" level=debug msg="Creating server traefik-ingress-controller-lqkjn at https://11.22.33.44:8080 with weight 1"
time="2018-05-31T10:54:59Z" level=debug msg="Add certificate for domains mydomain.de,traefik.mydomain.de"
time="2018-05-31T10:54:59Z" level=info msg="Server configuration reloaded on :443"
time="2018-05-31T10:54:59Z" level=info msg="Server configuration reloaded on :80"
time="2018-05-31T10:54:59Z" level=info msg="Server configuration reloaded on :8080"
time="2018-05-31T10:54:59Z" level=debug msg="Try to challenge certificate for domain [traefik.mydomain.de] founded in Host rule"
time="2018-05-31T10:54:59Z" level=debug msg="No domain parsed in rule \"PathPrefix:/\""
time="2018-05-31T10:54:59Z" level=debug msg="Looking for provided certificate(s) to validate [\"traefik.mydomain.de\"]..."
time="2018-05-31T10:54:59Z" level=debug msg="No ACME certificate to generate for domains [\"traefik.mydomain.de\"]."
time="2018-05-31T10:54:59Z" level=debug msg="Received Kubernetes event kind *v1.Secret"
time="2018-05-31T10:54:59Z" level=debug msg="Skipping Kubernetes event kind *v1.Secret"
time="2018-05-31T10:54:59Z" level=debug msg="Received Kubernetes event kind *v1.Secret"
<<< many more skipped events >>>
time="2018-05-31T10:55:16Z" level=debug msg="Skipping Kubernetes event kind *v1.Endpoints"
time="2018-05-31T10:55:16Z" level=debug msg="Received Kubernetes event kind *v1.Endpoints"
time="2018-05-31T10:55:16Z" level=debug msg="Skipping Kubernetes event kind *v1.Endpoints"
<<< many more skipped events >>>
Unfortunately I'm lacking required debugging skills to analyse much further. I checked that my configured config files are available and readable. I checked that the acme.json
is being used - it contains informations about the issued certificates.
Note: I tried my best to keep this readable and as short as possible (without leaving out important information) but was most likely not able to keep it as minimal as it should be. Pardon me for that - asking questions is way harder when you are not yet firm in a topic.
After a lot of research: There were (at least) 2 errors in my configuration.
defaultEntryPoints
. It seems that the Traefik Web Ui does not configure an endpoint for the frontend (at least not by itself). As a result the web ui was not reachable in my configuration. Adding the default entrypoints (which are used when there is no specific frontend entrypoint configuration) solves that issue.Solution: Add this line in the traefik.toml
definition for the ConfigMap
named traefik-config
(see the full configuration below):
defaultEntryPoints = ["http", "https"]
80
to the internal service port 8080
that the Traefik web ui uses. This can be changed by updating the configuration of the Service
named traefik-web-ui
and adding targetPort: 8080
(see the complete configuration below).After these changes my setup works as intended.
The complete config just for reference:
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
rules:
- apiGroups:
- ""
resources:
- services
- endpoints
- secrets
verbs:
- get
- list
- watch
- apiGroups:
- extensions
resources:
- ingresses
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: traefik-ingress-controller
subjects:
- kind: ServiceAccount
name: traefik-ingress-controller
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: traefik-ingress-controller
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: traefik-config
namespace: kube-system
data:
traefik.toml: |
defaultEntryPoints = ["http", "https"]
[entryPoints]
[entryPoints.http]
address = ":80"
[entryPoints.http.redirect]
entryPoint = "https"
[entryPoints.https]
address = ":443"
[entryPoints.https.tls]
[acme]
email = "admin@mydomain.de"
storage = "/acme/acme.json"
onHostRule = true
caServer = "https://acme-staging-v02.api.letsencrypt.org/directory"
entryPoint = "https"
[acme.httpChallenge]
entryPoint = "http"
[[acme.domains]]
main = "mydomain.de"
sans = ["traefik.mydomain.de"]
---
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
name: traefik-ingress-controller
namespace: kube-system
labels:
k8s-app: traefik-ingress-lb
spec:
template:
metadata:
labels:
k8s-app: traefik-ingress-lb
name: traefik-ingress-lb
spec:
serviceAccountName: traefik-ingress-controller
terminationGracePeriodSeconds: 60
hostNetwork: true
volumes:
- name: config
configMap:
name: traefik-config
- name: acme
hostPath:
path: /srv/configs/acme.json
type: File
containers:
- image: traefik
name: traefik-ingress-lb
volumeMounts:
- mountPath: "/config"
name: "config"
- mountPath: "/acme/acme.json"
name: "acme"
ports:
- name: http
containerPort: 80
hostPort: 80
- name: https
containerPort: 443
hostPort: 443
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
args:
- --configfile=/config/traefik.toml
- --api
- --kubernetes
- --logLevel=DEBUG
---
kind: Service
apiVersion: v1
metadata:
name: traefik-ingress-service
namespace: kube-system
spec:
selector:
k8s-app: traefik-ingress-lb
ports:
- protocol: TCP
port: 80
name: http
- protocol: TCP
port: 443
name: https
type: NodePort
---
apiVersion: v1
kind: Service
metadata:
name: traefik-web-ui
namespace: kube-system
spec:
selector:
k8s-app: traefik-ingress-lb
ports:
- protocol: TCP
port: 80
targetPort: 8080
name: webui
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: traefik-web-ui
namespace: kube-system
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: traefik.mydomain.de
http:
paths:
- path: /
backend:
serviceName: traefik-web-ui
servicePort: 80