Мобильная версияКод: Выделить всё
2024-10-31 15:31:07,952 ERROR [Timer-Driven Process Thread-1] o.apache.nifi.processors.hadoop.PutHDFS PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] Failed to write to HDFS
org.apache.nifi.processor.exception.ProcessException: IOException thrown from PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd]: org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /pakistan/.db7bd26c-6ccc-4dcf-88f0-fc3965286fc3 could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and 1 node(s) are excluded in this operation.
at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1641)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getNewBlockTargets(FSNamesystem.java:3198)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3122)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:843)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:500)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:640)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2313)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2309)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2307)
at org.apache.nifi.controller.repository.StandardProcessSession.read(StandardProcessSession.java:2711)
at org.apache.nifi.controller.repository.StandardProcessSession.read(StandardProcessSession.java:2656)
at org.apache.nifi.processors.hadoop.PutHDFS$1.run(PutHDFS.java:405)
at java.base/java.security.AccessController.doPrivileged(AccessController.java:400)
at java.base/javax.security.auth.Subject.doAs(Subject.java:453)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
at org.apache.nifi.processors.hadoop.PutHDFS.onTrigger(PutHDFS.java:328)
at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27)
at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1361)
at org.apache.nifi.controller.tasks.ConnectableTask.invoke(ConnectableTask.java:247)
at org.apache.nifi.controller.scheduling.AbstractTimeBasedSchedulingAgent.lambda$doScheduleOnce$0(AbstractTimeBasedSchedulingAgent.java:59)
at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
Caused by: org.apache.hadoop.ipc.RemoteException: File /pakistan/.db7bd26c-6ccc-4dcf-88f0-fc3965286fc3 could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and 1 node(s) are excluded in this operation.
at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1641)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getNewBlockTargets(FSNamesystem.java:3198)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3122)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:843)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:500)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:640)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2313)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2309)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2307)
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584)
at org.apache.hadoop.ipc.Client.call(Client.java:1529)
at org.apache.hadoop.ipc.Client.call(Client.java:1426)
at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258)
at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139)
at jdk.proxy7/jdk.proxy7.$Proxy171.addBlock(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500)
at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437)
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170)
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162)
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366)
at jdk.proxy7/jdk.proxy7.$Proxy172.addBlock(Unknown Source)
at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143)
at org.apache.hadoop.hdfs.DataStreamer.locateFollowingBlock(DataStreamer.java:2009)
at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1811)
at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:751)
2024-10-31 15:31:07,958 INFO [Timer-Driven Process Thread-1] o.a.n.c.s.StandardProcessScheduler Stopping PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd]
2024-10-31 15:31:07,958 INFO [Timer-Driven Process Thread-1] o.a.n.controller.StandardProcessorNode Stopping processor: PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd]
2024-10-31 15:31:07,959 INFO [Timer-Driven Process Thread-1] o.a.n.c.s.TimerDrivenSchedulingAgent Stopped scheduling PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] to run
2024-10-31 15:31:07,960 WARN [org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner] org.apache.hadoop.fs.FileSystem Cleaner thread interrupted, will stop
java.lang.InterruptedException: n u l l < b r / > < / c o d e > < b r / > н и ж е п о к а з а н о и з о б р а ж е н и е г р у п п ы п р о ц е с с о р о в с о ш и б к о й н а х о л с т е . < / p > < b r / > < i m g a l t = " г р у п п а п р о ц е с с о р о в п л ю с о ш и б к а " s r c = " h t t p s : / / i . s s t a t i c . n e t / o J O p V p A 4 . p n g " / > < / p > < b r / > з д е с ь н и ж е п о к а з а н о и з о б р а ж е н и е к о н ф и г у р а ц и и п р о ц е с с о р а p u t h d f s < / c o d e > . < / p > < b r / > < i m g a l t = " К о н ф и г у р а ц и и п р о ц е с с о р а p u t h d f s " s r c = " h t t p s : / / i . s s t a t i c . n e t / v T J P S h O o . p n g " / > < / p > < b r / > Т а к ж е я п р и л а г а ю ф а й л c o r e - s i t e . x m l < / c o d e > < / p > < b r / > < c o d e > & l t ; c o n f i g u r a t i o n & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; f s . d e f a u l t F S & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; h d f s : / / 1 9 2 . 1 6 8 . 1 3 6 . 1 3 1 : 8 0 2 0 & l t ; / v a l u e & g t ; < b r / > & l t ; f i n a l & g t ; t r u e & l t ; / f i n a l & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; f s . t r a s h . i n t e r v a l & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; 3 6 0 & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a . f a i l o v e r - c o n t r o l l e r . a c t i v e - s t a n d b y - e l e c t o r . z k . o p . r e t r i e s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; 1 2 0 & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . h t t p . a u t h e n t i c a t i o n . s i m p l e . a n o n y m o u s . a l l o w e d & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; t r u e & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . f a l c o n . g r o u p s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . f a l c o n . h o s t s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h b a s e . g r o u p s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h b a s e . h o s t s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h c a t . g r o u p s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ;
hadoop.proxyuser.hcat.hosts
sandbox.hortonworks.com
hadoop.proxyuser.hdfs.groups
*
hadoop.proxyuser.hdfs.hosts
*
hadoop.proxyuser.hive.groups
*
hadoop.proxyuser.hive.hosts
sandbox.hortonworks.com
hadoop.proxyuser.hue.groups
*
hadoop.proxyuser.hue.hosts
*
hadoop.proxyuser.livy.groups
*
hadoop.proxyuser.livy.hosts
*
hadoop.proxyuser.oozie.groups
*
hadoop.proxyuser.oozie.hosts
sandbox.hortonworks.com
hadoop.proxyuser.root.groups
*
hadoop.proxyuser.root.hosts
sandbox.hortonworks.com
hadoop.security.auth_to_local
DEFAULT
hadoop.security.authentication
simple
hadoop.security.authorization
false
hadoop.security.key.provider.path
io.compression.codec.lzo.class
com.hadoop.compression.lzo.LzoCodec
io.file.buffer.size
131072
io.serializations
org.apache.hadoop.io.serializer.WritableSerialization
ipc.client.connect.max.retries
50
ipc.client.connection.maxidletime
30000
ipc.client.idlethreshold
8000
ipc.server.tcpnodelay
true
mapreduce.jobtracker.webinterface.trusted
false
net.topology.script.file.name
/etc/hadoop/conf/topology_script.py
hadoop.proxyuser.nifi.groups
*
hadoop.proxyuser.nifi.hosts
*
Подробнее здесь: https://stackoverflow.com/questions/791 ... eplication
