Код: Выделить всё
2024-10-31 09:41:30,759 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Successfully checkpointed FlowFile Repository with 6 records in 8 milliseconds 2024-10-31 09:41:31,435 ERROR [Timer-Driven Process Thread-2] o.apache.nifi.processors.hadoop.PutHDFS PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] HDFS Configuration error org.apache.hadoop.net.ConnectTimeoutException: 1000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=/192.168.136.131:8020] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:604) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:564) at org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.checkHdfsUriForTimeout(AbstractHadoopProcessor.java:586) at org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.resetHDFSResources(AbstractHadoopProcessor.java:476) at org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.abstractOnScheduled(AbstractHadoopProcessor.java:387) at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) at java.base/java.lang.reflect.Method.invoke(Method.java:580) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:145) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:133) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:78) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotation(ReflectionUtils.java:55) at org.apache.nifi.controller.StandardProcessorNode.lambda$initiateStart$8(StandardProcessorNode.java:1765) at org.apache.nifi.engine.FlowEngine$3.call(FlowEngine.java:123) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1570) 2024-10-31 09:41:31,436 ERROR [Timer-Driven Process Thread-2] o.apache.nifi.processors.hadoop.PutHDFS PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] Failed to properly initialize Processor. If still scheduled to run, NiFi will attempt to initialize and run the Processor again after the 'Administrative Yield Duration' has elapsed. Failure is due to org.apache.hadoop.net.ConnectTimeoutException: 1000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=/192.168.136.131:8020] org.apache.hadoop.net.ConnectTimeoutException: 1000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=/192.168.136.131:8020] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:604) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:564) at org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.checkHdfsUriForTimeout(AbstractHadoopProcessor.java:586) at org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.resetHDFSResources(AbstractHadoopProcessor.java:476) at org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.abstractOnScheduled(AbstractHadoopProcessor.java:387) at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) at java.base/java.lang.reflect.Method.invoke(Method.java:580) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:145) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:133) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:78) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotation(ReflectionUtils.java:55) at org.apache.nifi.controller.StandardProcessorNode.lambda$initiateStart$8(StandardProcessorNode.java:1765) at org.apache.nifi.engine.FlowEngine$3.call(FlowEngine.java:123) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1570) 2024-10-31 09:41:31,436 WARN [Timer-Driven Process Thread-2] o.a.n.p.hadoop.HDFSResourceHelper Error stopping FileSystem statistics thread: null 2024-10-31 09:41:34,582 INFO [NiFi Web Server-57] o.a.n.c.s.StandardProcessScheduler Stopping PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] 2024-10-31 09:41:34,582 INFO [NiFi Web Server-57] o.a.n.controller.StandardProcessorNode Stopping processor: PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] 2024-10-31 09:41:34,583 INFO [NiFi Web Server-57] o.a.n.c.s.StandardProcessScheduler Stopping GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc] 2024-10-31 09:41:34,583 INFO [NiFi Web Server-57] o.a.n.controller.StandardProcessorNode Stopping processor: GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc] 2024-10-31 09:41:34,583 INFO [Timer-Driven Process Thread-5] o.a.n.c.s.TimerDrivenSchedulingAgent Stopped scheduling GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc] to run 2024-10-31 09:41:34,584 INFO [Timer-Driven Process Thread-5] o.a.n.controller.StandardProcessorNode GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc] has completely stopped. Completing any associated Futures. 2024-10-31 09:41:35,078 INFO [Flow Service Tasks Thread-1] o.a.nifi.controller.StandardFlowService Saved flow controller org.apache.nifi.controller.FlowController@162d5d5e // Another save pending = false

здесь ниже показан весь поток данных группы процессоров вместе с ошибкой:
< img alt="процессорный групповой поток" src="https://i.sstatic.net/7o45X1Le.png" />
также я добавляю минималистичный файл core-site.xml< /code> конфигурации, которые я импортировал из HDP 2.5, поместил в каталог /conf и настроил его путь вместе с hdfs-site.xml в процессоре puthdfs .
Код: Выделить всё
enter code here
hdfs://192.168.136.131:8020
true
fs.trash.interval
360
ha.failover-controller.active-standby-elector.zk.op.retries
120
Hadoop.http.authentication.simple.anonymous.allowed true
Hadoop.proxyuser.falcon.groups
*
Hadoop.proxyuser.falcon.hosts
*
Hadoop.proxyuser.hbase.groups
*
Hadoop.proxyuser.hbase.hosts
*
Hadoop.proxyuser.hcat.groups
*
Hadoop.proxyuser.hcat.hosts
sandbox.hortonworks.com
Hadoop.proxyuser.hdfs.groups
*
Hadoop.proxyuser.hdfs.hosts
*
Hadoop.proxyuser.hive.groups
*
Hadoop.proxyuser.hive.hosts
sandbox.hortonworks.com
Hadoop.proxyuser.hue.groups
*
Hadoop.proxyuser.hue.hosts
*
Hadoop.proxyuser.livy.groups
*
Hadoop.proxyuser.livy.hosts
*
Hadoop.proxyuser.oozie.groups
*
Hadoop.proxyuser.oozie.hosts sandbox.hortonworks.com
Hadoop.proxyuser.root.groups
*
Hadoop.proxyuser.root.hosts
sandbox.hortonworks.com
Hadoop.security.auth_to_local
ПО УМОЛЧАНИЮ
Hadoop.security.authentication
простой
Hadoop.security.authorization
false
Hadoop.security.key.provider.path
io. compress.codec.lzo.class
com.hadoop.compression.lzo.LzoCodec
io.compression.codecs
org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop. compress.lzo.LzopCodec
io.file.buffer.size
131072
io.serializations
org.apache.hadoop.io.serializer.WritableSerialization
ipc.client.connect.max.retries
50
ipc.client.connection.maxidletime
30000
ipc.client.idlethreshold
8000
ipc.server.tcpnodelay
true
mapreduce.jobtracker.webinterface.trusted
false
net.topology.script.file.name
/etc/hadoop/conf/topology_script. py
Hadoop.proxyuser.nifi.groups
*
Hadoop.proxyuser.nifi.hosts
*
Я буду благодарен, если кто-нибудь поможет мне, где я делаю неправильно.
Подробнее здесь: https://stackoverflow.com/questions/791 ... lconnectio