Some jobs get aborted due to exception:
java.io.IOException: Read end dead at java.io.PipedInputStream.checkStateForReceive(PipedInputStream.java:262) at java.io.PipedInputStream.awaitSpace(PipedInputStream.java:268) at java.io.PipedInputStream.receive(PipedInputStream.java:231) at java.io.PipedOutputStream.write(PipedOutputStream.java:149) at java.io.OutputStream.write(OutputStream.java:75) at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.setupEnvironmentVariable(ContainerExecDecorator.java:424) at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:388) at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:246) at hudson.Launcher$ProcStarter.start(Launcher.java:455) at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:206) at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:99) at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:305) at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:268) at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:176) at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122) at sun.reflect.GeneratedMethodAccessor5839.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1213) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022) at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42) at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48) at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113) at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:157) at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23) at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:155) at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:155) at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:159) at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129) at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129) at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17) at org.centos.pipeline.PipelineUtils.executeInContainer(file:/var/lib/jenkins/jobs/fedora-rawhide-pr-pipeline/builds/1269/libs/ci-pipeline/src/org/centos/pipeline/PipelineUtils.groovy:762) at ___cps.transform___(Native Method) at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:57) at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:109) at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82) at sun.reflect.GeneratedMethodAccessor258.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72) at com.cloudbees.groovy.cps.impl.LocalVariableBlock$LocalVariable.get(LocalVariableBlock.java:39) at com.cloudbees.groovy.cps.LValueBlock$GetAdapter.receive(LValueBlock.java:30) at com.cloudbees.groovy.cps.impl.LocalVariableBlock.evalLValue(LocalVariableBlock.java:28) at com.cloudbees.groovy.cps.LValueBlock$BlockImpl.eval(LValueBlock.java:55) at com.cloudbees.groovy.cps.LValueBlock.eval(LValueBlock.java:16) at com.cloudbees.groovy.cps.Next.step(Next.java:83) at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174) at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163) at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:129) at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:268) at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163) at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$101(SandboxContinuable.java:34) at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.lambda$run0$0(SandboxContinuable.java:59) at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.GroovySandbox.runInSandbox(GroovySandbox.java:121) at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:58) at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:182) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:332) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$200(CpsThreadGroup.java:83) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:244) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:232) at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:64) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:131) at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28) at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Also at same time there is another exception, that I'm not sure is related:
13:37:30 + mkdir -p koji-build [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh 13:37:38 java.net.SocketTimeoutException: sent ping but didn't receive pong within 1000ms (after 5 successful ping/pongs) 13:37:38 at okhttp3.internal.ws.RealWebSocket.writePingFrame(RealWebSocket.java:546) 13:37:38 at okhttp3.internal.ws.RealWebSocket$PingRunnable.run(RealWebSocket.java:530) 13:37:38 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 13:37:38 at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) 13:37:38 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) 13:37:38 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) 13:37:38 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 13:37:38 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 13:37:38 at java.lang.Thread.run(Thread.java:748) [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] sh 13:37:38 + '[' -d logs ']' 13:37:38 + echo 'No logs for executeInContainer(). Ignoring this.' 13:37:38 No logs for executeInContainer(). Ignoring this. [Pipeline] echo 13:37:38 Read end dead
logs from: https://jenkins-continuous-infra.apps.ci.centos.org/job/fedora-rawhide-pr-pipeline/1269/console
@bstinson Is it possible to get log from a jenkins slave for such failures?
Metadata Update from @bookwar: - Issue assigned to bstinson
Also I wonder, which version of plugins do we have?
For example, there is an issue like this, which may ot may not be relevant https://github.com/jenkinsci/kubernetes-plugin/pull/442
And also there are several updates to pipelines plugin which add some new features like https://issues.jenkins-ci.org/browse/JENKINS-26133 which could help us simplify the pipeline.
Can we update Jenkins master to everything latest and greatest, to make sure that we don't hit some old issues which might be resolved already
Any update on this? We still experience this issue.
22:35:21 + mkdir -p koji-build [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh 22:35:25 java.net.SocketTimeoutException: sent ping but didn't receive pong within 1000ms (after 1 successful ping/pongs) 22:35:25 at okhttp3.internal.ws.RealWebSocket.writePingFrame(RealWebSocket.java:546) 22:35:25 at okhttp3.internal.ws.RealWebSocket$PingRunnable.run(RealWebSocket.java:530) 22:35:25 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 22:35:25 at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) 22:35:25 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) 22:35:25 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) 22:35:25 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 22:35:25 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 22:35:25 at java.lang.Thread.run(Thread.java:748) 22:40:31 process apparently never started in /workDir/workspace/fedora-rawhide-build-pipeline@tmp/durable-53630e6f [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] sh 22:40:31 + '[' -d logs ']' 22:40:31 + echo 'No logs for executeInContainer(). Ignoring this.' 22:40:31 No logs for executeInContainer(). Ignoring this. [Pipeline] echo 22:40:31 script returned exit code -2
https://jenkins-continuous-infra.apps.ci.centos.org/job/fedora-rawhide-build-pipeline/4672/console
Metadata Update from @bookwar: - Issue tagged with: jenkins
@bookwar I haven't see this recently, can I just close it?
Logs have rotated out and I haven't seen this lately, so closing this out. We can re-open if we find it happening again.
Metadata Update from @jimbair: - Issue status updated to: Closed (was: Open)
Login to comment on this ticket.