前两天一直dataframe写csv写不出来，报有的node refuse connect等等 看样子是集群的问题。

然后将dataframe换为rdd，增大一倍excutors个数，8分钟跑完。。。

dataframe搞不定，那就rdd + 资源吧。

记录下报错：

18/11/26 10:04:09 ERROR TransportResponseHandler: Still have 1 requests outstanding when connection from node19-69-189-bdyf.qiyi.hadoop/10.19.69.189:39473 is closed

18/11/26 10:04:09 ERROR OneForOneBlockFetcher: Failed while starting block fetches

java.io.IOException: Connection reset by peer

at sun.nio.ch.FileDispatcherImpl.read0(Native Method)

at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)

at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)

at sun.nio.ch.IOUtil.read(IOUtil.java:192)

at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)

at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:221)

at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:899)

at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:275)

at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:119)

at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:652)

at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:575)

at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:489)

at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:451)

at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:140)

at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)

at java.lang.Thread.run(Thread.java:745)

18/11/26 10:04:10 ERROR TransportResponseHandler: Still have 4 requests outstanding when connection from node19-67-108-bdyf.qiyi.hadoop/10.19.67.108:37395 is closed

18/11/26 10:04:10 ERROR OneForOneBlockFetcher: Failed while starting block fetches

java.io.IOException: Connection reset by peer

at sun.nio.ch.FileDispatcherImpl.read0(Native Method)

at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)

at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)

at sun.nio.ch.IOUtil.read(IOUtil.java:192)

at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)

at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:221)

at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:899)

at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:275)

at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:119)

at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:652)

at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:575)

at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:489)

at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:451)

at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:140)

at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)

at java.lang.Thread.run(Thread.java:745)

18/11/26 10:04:10 ERROR OneForOneBlockFetcher: Failed while starting block fetches

java.io.IOException: Connection reset by peer

at sun.nio.ch.FileDispatcherImpl.read0(Native Method)

at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)

at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)

at sun.nio.ch.IOUtil.read(IOUtil.java:192)

at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)

at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:221)

at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:899)

at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:275)

at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:119)

at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:652)

at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:575)

at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:489)

at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:451)

at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:140)

at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)

at java.lang.Thread.run(Thread.java:745)

18/11/26 10:04:10 ERROR OneForOneBlockFetcher: Failed while starting block fetches

java.io.IOException: Connection reset by peer

at sun.nio.ch.FileDispatcherImpl.read0(Native Method)

at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)

at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)

at sun.nio.ch.IOUtil.read(IOUtil.java:192)

at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)

at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:221)

at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:899)

at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:275)

at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:119)

at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:652)

at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:575)

at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:489)

at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:451)

at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:140)

at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)

at java.lang.Thread.run(Thread.java:745)