I have updated the the kubernetes plugins and the kubernetes-credentials plugins from:
'kubernetes:1.18.3',
'kubernetes-credentials:0.4.0'
to
'kubernetes:1.25.3'
'kubernetes-client-api:4.9.1-1'
'kubernetes-credentials:0.6.2'
and now I can see the following exception in the Jenkins logs. The log is littered with this warning message. I am able to run all jobs as expected, and without any issues. We have two kubernetes clusters that we are using with jenkins. Jenkins is running on a Bare metal cluster with kubernetes version v1.16.3.
2020-04-27 08:20:25.450+0000 [id=71902] INFO o.c.j.p.k.KubernetesClientProvider#gracefulClose: Not closing io.fabric8.kubernetes.client.DefaultKubernetesClient@7274903e: there are still running (1) or queued (0) calls
2020-04-27 08:20:37.054+0000 [id=71903] INFO hudson.model.AsyncPeriodicWork#lambda$doRun$0: Started prometheus_async_worker
2020-04-27 08:20:37.178+0000 [id=71903] INFO hudson.model.AsyncPeriodicWork#lambda$doRun$0: Finished prometheus_async_worker. 124 ms
2020-04-27 08:20:44.904+0000 [id=2857] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure
java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@1d4a676e rejected from java.util.concurrent.ThreadPoolExecutor@543ef1af[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 18]
at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)
at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)
Caused: java.io.InterruptedIOException: executor rejected
at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)
at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)
at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)
at okhttp3.RealCall.enqueue(RealCall.java:127)
at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)
at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:158)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:321)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:410)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2020-04-27 08:21:16.904+0000 [id=2857] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure
java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@cfe8b6d rejected from java.util.concurrent.ThreadPoolExecutor@543ef1af[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 18]
at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)
at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)
Caused: java.io.InterruptedIOException: executor rejected
at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)
at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)
at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)
at okhttp3.RealCall.enqueue(RealCall.java:127)
at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)
at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:158)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:321)
at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:410)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)