Мобильная версияКод: Выделить всё
2024-10-31 12:19:20,860 ERROR [Timer-Driven Process Thread-10] o.apache.nifi.processors.hadoop.PutHDFS PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] Failed to write to HDFS
java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configurable
at java.base/java.lang.ClassLoader.defineClass1(Native Method)
at java.base/java.lang.ClassLoader.defineClass(ClassLoader.java:1023)
at java.base/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
at java.base/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
at java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:760)
at java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:579)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:579)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:579)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:579)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
at java.base/java.lang.Class.forName0(Native Method)
at java.base/java.lang.Class.forName(Class.java:529)
at java.base/java.lang.Class.forName(Class.java:508)
at org.apache.nifi.processors.hadoop.ExtendedConfiguration.getClassByNameOrNull(ExtendedConfiguration.java:70)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2617)
at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:132)
at org.apache.hadoop.io.compress.CompressionCodecFactory.(CompressionCodecFactory.java:182)
at org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.getCompressionCodec(AbstractHadoopProcessor.java:605)
at org.apache.nifi.processors.hadoop.PutHDFS$1.run(PutHDFS.java:341)
at java.base/java.security.AccessController.doPrivileged(AccessController.java:400)
at java.base/javax.security.auth.Subject.doAs(Subject.java:453)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
at org.apache.nifi.processors.hadoop.PutHDFS.onTrigger(PutHDFS.java:328)
at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27)
at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1361)
at org.apache.nifi.controller.tasks.ConnectableTask.invoke(ConnectableTask.java:247)
at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:102)
at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configurable
at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
... 37 common frames omitted
2024-10-31 12:19:21,002 INFO [Write-Ahead Local State Provider Maintenance] org.wali.MinimalLockingWriteAheadLog org.wali.MinimalLockingWriteAheadLog@2cb95c6c checkpointed with 2 Records and 0 Swap Files in 49 milliseconds (Stop-the-world time = 21 milliseconds, Clear Edit Logs time = 14 millis), max Transaction ID 105
2024-10-31 12:19:24,899 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Initiating checkpoint of FlowFile Repository
2024-10-31 12:19:24,908 INFO [pool-7-thread-1] o.a.n.wali.SequentialAccessWriteAheadLog Checkpointed Write-Ahead Log with 4 Records and 0 Swap Files in 9 milliseconds (Stop-the-world time = 4 milliseconds), max Transaction ID 292
2024-10-31 12:19:24,909 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Successfully checkpointed FlowFile Repository with 4 records in 9 milliseconds
2024-10-31 12:19:29,755 INFO [NiFi Web Server-41] o.a.n.c.s.StandardProcessScheduler Stopping PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd]
2024-10-31 12:19:29,755 INFO [NiFi Web Server-41] o.a.n.controller.StandardProcessorNode Stopping processor: PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd]
2024-10-31 12:19:29,757 INFO [Timer-Driven Process Thread-10] o.a.n.c.s.TimerDrivenSchedulingAgent Stopped scheduling PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] to run
2024-10-31 12:19:29,757 INFO [NiFi Web Server-41] o.a.n.c.s.StandardProcessScheduler Stopping GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc]
2024-10-31 12:19:29,757 INFO [NiFi Web Server-41] o.a.n.controller.StandardProcessorNode Stopping processor: GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc]
2024-10-31 12:19:29,757 INFO [Timer-Driven Process Thread-3] o.a.n.c.s.TimerDrivenSchedulingAgent Stopped scheduling GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc] to run
2024-10-31 12:19:29,758 INFO [Timer-Driven Process Thread-3] o.a.n.controller.StandardProcessorNode GenerateFlowFile[id=d2b75046-0192-1000-10ed-4952d26261bc] has completely stopped. Completing any associated Futures.
2024-10-31 12:19:29,762 WARN [org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner] org.apache.hadoop.fs.FileSystem Cleaner thread interrupted, will stop
java.lang.InterruptedException: null
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1722)
at java.base/java.lang.ref.ReferenceQueue.await(ReferenceQueue.java:67)
at java.base/java.lang.ref.ReferenceQueue.remove0(ReferenceQueue.java:158)
at java.base/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:234)
at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4157)
at java.base/java.lang.Thread.run(Thread.java:1570)
2024-10-31 12:19:29,763 INFO [Timer-Driven Process Thread-10] o.a.n.controller.StandardProcessorNode PutHDFS[id=d2b7ad77-0192-1000-e937-9d1797c4e0fd] has completely stopped. Completing any associated Futures.
2024-10-31 12:19:30,327 INFO [Flow Service Tasks Thread-2] o.a.nifi.controller.StandardFlowService Saved flow controller org.apache.nifi.controller.FlowController@7e46c2ea // Another save pending = false
2024-10-31 12:19:44,914 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Initiating checkpoint of FlowFile Repository
2024-10-31 12:19:44,914 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Successfully checkpointed FlowFile Repository with 4 records in 0 milliseconds
2024-10-31 12:20:04,921 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Initiating checkpoint of FlowFile Repository
2024-10-31 12:20:04,921 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Successfully checkpointed FlowFile Repository with 4 records in 0 milliseconds
2024-10-31 12:20:20,880 INFO [Cleanup Archive for default] o.a.n.c.repository.FileSystemRepository Successfully deleted 0 files (0 bytes) from archive
2024-10-31 12:20:20,881 INFO [Cleanup Archive for default] o.a.n.c.repository.FileSystemRepository Archive cleanup completed for container default; will now allow writing to this container. Bytes used = 184.4 GB, bytes free = 1.02 GB, capacity = 185.42 GB
2024-10-31 12:20:24,934 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Initiating checkpoint of FlowFile Repository
2024-10-31 12:20:24,934 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Successfully checkpointed FlowFile Repository with 4 records in 0 milliseconds
2024-10-31 12:20:44,938 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Initiating checkpoint of FlowFile Repository
2024-10-31 12:20:44,938 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Successfully checkpointed FlowFile Repository with 4 records in 0 milliseconds
2024-10-31 12:21:04,949 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Initiating checkpoint of FlowFile Repository
2024-10-31 12:21:04,950 INFO [pool-7-thread-1] o.a.n.c.r.WriteAheadFlowFileRepository Successfully checkpointed FlowFile Repository with 4 records in 0 milliseconds
2024-10-31 12:21:20,921 INFO [Cleanup Archive for default] o.a.n.c.repository.FileSystemRepository Successfully deleted 0 files (0 bytes) from archive
2024-10-31 12:21:20,922 INFO [Cleanup Archive for default] o.a.n.c.repository.FileSystemRepository Archive cleanup completed for container default; will now allow writing to
[img]https://i. sstatic.net/JpsFWsb2.png[/img]
ниже показан весь поток данных группы процессоров вместе с ошибкой:
также я добавляю минималистичные конфигурации core-site.xml, которые я импортировали из HDP 2.5, п о м е с т и л и в к а т а л о г / c o n f < / c o d e > и н а с т р о и л и е г о п у т ь в м е с т е с h d f s - s i t e . x m l < / c o d e > в п р о ц е с с о р е p u t h d f s < / c o d e > . < / p > < b r / > < c o d e > & l t ; c o n f i g u r a t i o n & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; f s . d e f a u l t F S & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; h d f s : / / 1 9 2 . 1 6 8 . 1 3 6 . 1 3 1 : 8 0 2 0 & l t ; / v a l u e & g t ; < b r / > & l t ; f i n a l & g t ; t r u e & l t ; / f i n a l & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; f s . t r a s h . i n t e r v a l & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; 3 6 0 & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a . f a i l o v e r - c o n t r o l l e r . a c t i v e - s t a n d b y - e l e c t o r . z k . o p . r e t r i e s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; 1 2 0 & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . h t t p . a u t h e n t i c a t i o n . s i m p l e . a n o n y m o u s . a l l o w e d & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; t r u e & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . f a l c o n . g r o u p s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . f a l c o n . h o s t s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h b a s e . g r o u p s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h b a s e . h o s t s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h c a t . g r o u p s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h c a t . h o s t s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; s a n d b o x . h o r t o n w o r k s . c o m & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r t y & g t ; < b r / > < b r / > & l t ; p r o p e r t y & g t ; < b r / > & l t ; n a m e & g t ; h a d o o p . p r o x y u s e r . h d f s . g r o u p s & l t ; / n a m e & g t ; < b r / > & l t ; v a l u e & g t ; * & l t ; / v a l u e & g t ; < b r / > & l t ; / p r o p e r ty>
hadoop.proxyuser.hdfs.hosts
*
hadoop.proxyuser.hive.groups
*
hadoop.proxyuser.hive.hosts
sandbox.hortonworks.com
hadoop.proxyuser.hue.groups
*
hadoop.proxyuser.hue.hosts
*
hadoop.proxyuser.livy.groups
*
hadoop.proxyuser.livy.hosts
*
hadoop.proxyuser.oozie.groups
*
hadoop.proxyuser.oozie.hosts
sandbox.hortonworks.com
hadoop.proxyuser.root.groups
*
hadoop.proxyuser.root.hosts
sandbox.hortonworks.com
hadoop.security.auth_to_local
DEFAULT
hadoop.security.authentication
simple
hadoop.security.authorization
false
hadoop.security.key.provider.path
io.compression.codec.lzo.class
com.hadoop.compression.lzo.LzoCodec
io.compression.codecs
org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec
io.file.buffer.size
131072
io.serializations
org.apache.hadoop.io.serializer.WritableSerialization
ipc.client.connect.max.retries
50
ipc.client.connection.maxidletime
30000
ipc.client.idlethreshold
8000
ipc.server.tcpnodelay
true
mapreduce.jobtracker.webinterface.trusted
false
net.topology.script.file.name
/etc/hadoop/conf/topology_script.py
hadoop.proxyuser.nifi.groups
*
hadoop.proxyuser.nifi.hosts
*
Буду благодарен, если кто-нибудь поможет мне, где я делаю неправильно.
Подробнее здесь: https://stackoverflow.com/questions/791 ... -java-lang
