We are seeing timeout exceptions when we setup tomcat in cluster mode in Kubernetes. We tried increasing the connectTimeout and readTimeout values to 10000ms as 1000ms is default value, but we still see the issue.
Tomcat version : 9.0.43
25-Oct-2021 05:59:05.252 SEVERE [Catalina-utility-2] org.apache.catalina.tribes.membership.cloud.KubernetesMembershipProvider.fetchMembers Failed to open stream
java.io.IOException: Failed connection to https://100.127.240.1:443/api/v1/namespaces/dev-idms-ecm-ns/pods?labelSelector=app%3Didmsrepo-apphost with token eyJhbGciOiJSUzI1NiIsImtpZCI6I
i1QaGlMNnRnZTRmNzd6akE5ejRseEJoaHVrbElsbmVXYng1MkR0dTdZT1kifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZXYtaWRtcy1lY20tbnMiLCJrdW
Jlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoiaWRtc3JlcG8tYXBwaG9zdC1zZXJ2aWNlLWFjY291bnQtdG9rZW4tcmh2djgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiaWRtc3
JlcG8tYXBwaG9zdC1zZXJ2aWNlLWFjY291bnQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJjMjAzZWUxOS01ZjJjLTRiYTctYjFiOC05N2EyYjA4NTQ5MGYiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY2
91bnQ6ZGV2LWlkbXMtZWNtLW5zOmlkbXNyZXBvLWFwcGhvc3Qtc2VydmljZS1hY2NvdW50In0.d5kRR2V66g0oTceDllkvsZGgihlZCPO7eoBtTyQt4r2cu8XN87rv0iLhzyoVxmAU17_Lp7JjIOhbVqc6hEn2g3y1dYHWpwEe4lC5g_0Wf4Rxs13gur3W8Q
5wfKvVlj3sMeKTd_BGlQFMj1rP27LdvyGTRVb50nDXwrFGqkuODKbVRZkdDq5dD5U3QJNf9svtt0iINqqPQrUF7AmOsJK91JpprHnlRjKo59e_g-t0tO91WwJeifXysWhHFWy5_RxbBfQHn4ABLlvjyxrfJLZNEOwySmIOhUbK2QXIjbBuvKC9UX5nzPzNl0
it1kdUhvo7Z_YlDSkb3NHA21AfpeNTkjYvC7Y_KJF_hUKMZTUkvjFDiD7fCkaWaCMMfpI_W6pPtuuVXVmf2_FRqsAzzRYM9u9FMswzWplCY8_u77f6v0jb_u6G9eGmt1-F7VV74mQwJ9e3I2C8-Y78XTAvE75lZCD_CEd8k735bGZAswGZiU74zn-lLaovTl
ybVW6fOSmMw5VVe5U_rne81REgZAu1gCm-EFiS3wkTzj6uSZEzEGGG22P0GJtGHOIullWV1gmNCAWGTaxJhpuhWQzmCZsXgY8lPtrqc9QWqCV78q57fMUrEt0PM_qZgyiw3WrobYiIt-NpDQvCAy5tt_Hp3kGSYpEwiwmo-p6sdTEhhcljzv0
at org.apache.catalina.tribes.membership.cloud.TokenStreamProvider.openStream(TokenStreamProvider.java:57)
at org.apache.catalina.tribes.membership.cloud.KubernetesMembershipProvider.fetchMembers(KubernetesMembershipProvider.java:136)
at org.apache.catalina.tribes.membership.cloud.CloudMembershipProvider.heartbeat(CloudMembershipProvider.java:127)
at org.apache.catalina.tribes.group.GroupChannel.heartbeat(GroupChannel.java:208)
at org.apache.catalina.tribes.group.GroupChannel$HeartbeatRunnable.run(GroupChannel.java:822)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.net.SocketTimeoutException: connect timed out
at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:399)
at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:242)
at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:224)
at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:403)
at java.base/java.net.Socket.connect(Socket.java:609)
at java.base/sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:289)
at java.base/sun.net.NetworkClient.doConnect(NetworkClient.java:177)
at java.base/sun.net.www.http.HttpClient.openServer(HttpClient.java:474)
at java.base/sun.net.www.http.HttpClient.openServer(HttpClient.java:569)
at java.base/sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:265)
at java.base/sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:372)
at java.base/sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
at java.base/sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1187)
at java.base/sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1081)
at java.base/sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1592)
at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1520)
at java.base/sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:250)
at org.apache.catalina.tribes.membership.cloud.AbstractStreamProvider.openStream(AbstractStreamProvider.java:111)
at org.apache.catalina.tribes.membership.cloud.TokenStreamProvider.openStream(TokenStreamProvider.java:54)
... 11 more
25-Oct-2021 05:59:05.253 INFO [Catalina-utility-2] org.apache.catalina.tribes.group.interceptors.TcpFailureDetector.memberDisappeared Received memberDisappeared[org.apache.catalina.tribes.membership.MemberImpl[tcp://10.25.9.112:4000,10.25.9.112,4000, alive=-343433, securePort=-1, UDP Port=-1, id={50 7 28 -13 -102 -62 -109 -109 -114 69 -15 22 -10 -13 -110 -87 }, payload={}, command={}, domain={}]] message. Will verify.
25-Oct-2021 05:59:05.255 INFO [Catalina-utility-2] org.apache.catalina.tribes.group.interceptors.TcpFailureDetector.memberDisappeared Verification complete. Member still alive[org.apache.catalina.tribes.membership.MemberImpl[tcp://10.25.9.112:4000,10.25.9.112,4000, alive=-343433, securePort=-1, UDP Port=-1, id={50 7 28 -13 -102 -62 -109 -109 -114 69 -15 22 -10 -13 -110 -87 }, payload={}, command={}, domain={}]]
25-Oct-2021 07:57:15.728 SEVERE [Catalina-utility-1] org.apache.catalina.tribes.membership.cloud.KubernetesMembershipProvider.fetchMembers Failed to open stream
java.io.IOException: Failed connection to https://100.127.240.1:443/api/v1/namespaces/dev-idms-ecm-ns/pods?labelSelector=app%3Didmsrepo-apphost with token eyJhbGciOiJSUzI1NiIsImtpZCI6I
i1QaGlMNnRnZTRmNzd6akE5ejRseEJoaHVrbElsbmVXYng1MkR0dTdZT1kifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZXYtaWRtcy1lY20tbnMiLCJrdW
Jlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoiaWRtc3JlcG8tYXBwaG9zdC1zZXJ2aWNlLWFjY291bnQtdG9rZW4tcmh2djgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiaWRtc3
JlcG8tYXBwaG9zdC1zZXJ2aWNlLWFjY291bnQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJjMjAzZWUxOS01ZjJjLTRiYTctYjFiOC05N2EyYjA4NTQ5MGYiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY2
91bnQ6ZGV2LWlkbXMtZWNtLW5zOmlkbXNyZXBvLWFwcGhvc3Qtc2VydmljZS1hY2NvdW50In0.d5kRR2V66g0oTceDllkvsZGgihlZCPO7eoBtTyQt4r2cu8XN87rv0iLhzyoVxmAU17_Lp7JjIOhbVqc6hEn2g3y1dYHWpwEe4lC5g_0Wf4Rxs13gur3W8Q
5wfKvVlj3sMeKTd_BGlQFMj1rP27LdvyGTRVb50nDXwrFGqkuODKbVRZkdDq5dD5U3QJNf9svtt0iINqqPQrUF7AmOsJK91JpprHnlRjKo59e_g-t0tO91WwJeifXysWhHFWy5_RxbBfQHn4ABLlvjyxrfJLZNEOwySmIOhUbK2QXIjbBuvKC9UX5nzPzNl0
it1kdUhvo7Z_YlDSkb3NHA21AfpeNTkjYvC7Y_KJF_hUKMZTUkvjFDiD7fCkaWaCMMfpI_W6pPtuuVXVmf2_FRqsAzzRYM9u9FMswzWplCY8_u77f6v0jb_u6G9eGmt1-F7VV74mQwJ9e3I2C8-Y78XTAvE75lZCD_CEd8k735bGZAswGZiU74zn-lLaovTl
ybVW6fOSmMw5VVe5U_rne81REgZAu1gCm-EFiS3wkTzj6uSZEzEGGG22P0GJtGHOIullWV1gmNCAWGTaxJhpuhWQzmCZsXgY8lPtrqc9QWqCV78q57fMUrEt0PM_qZgyiw3WrobYiIt-NpDQvCAy5tt_Hp3kGSYpEwiwmo-p6sdTEhhcljzv0
at org.apache.catalina.tribes.membership.cloud.TokenStreamProvider.openStream(TokenStreamProvider.java:57)
at org.apache.catalina.tribes.membership.cloud.KubernetesMembershipProvider.fetchMembers(KubernetesMembershipProvider.java:136)
at org.apache.catalina.tribes.membership.cloud.CloudMembershipProvider.heartbeat(CloudMembershipProvider.java:127)
at org.apache.catalina.tribes.group.GroupChannel.heartbeat(GroupChannel.java:208)
at org.apache.catalina.tribes.group.GroupChannel$HeartbeatRunnable.run(GroupChannel.java:822)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.base/java.net.SocketInputStream.socketRead0(Native Method)
at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:476)
at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:470)
at java.base/sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:160)
at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:110)
at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1408)
at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1314)
at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:440)
at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:411)
at java.base/sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:567)
at java.base/sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:185)
at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1592)
at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1520)
at java.base/sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:250)
at org.apache.catalina.tribes.membership.cloud.AbstractStreamProvider.openStream(AbstractStreamProvider.java:111)
at org.apache.catalina.tribes.membership.cloud.TokenStreamProvider.openStream(TokenStreamProvider.java:54)
... 11 moreAny help would be appreciated.
Thanks