Код: Выделить всё
data=ukb_df_data("test/bulk_strings.tsv")
data.createOrReplaceTempView("synthetic_biobank1")
data.write.saveAsTable("synthetic_biobank1")
Код: Выделить всё
D:\hadoop\binКод: Выделить всё
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
Cell In[8], line 3
1 data=ukb_df_data("test/bulk_strings.tsv")
2 data.createOrReplaceTempView("synthetic_biobank1")
----> 3 data.write.saveAsTable("synthetic_biobank1")
File c:\Users\kunal\miniconda3\Lib\site-packages\pyspark\sql\readwriter.py:1521, in DataFrameWriter.saveAsTable(self, name, format, mode, partitionBy, **options)
1519 if format is not None:
1520 self.format(format)
-> 1521 self._jwrite.saveAsTable(name)
File c:\Users\kunal\miniconda3\Lib\site-packages\py4j\java_gateway.py:1322, in JavaMember.__call__(self, *args)
1316 command = proto.CALL_COMMAND_NAME +\
1317 self.command_header +\
1318 args_command +\
1319 proto.END_COMMAND_PART
1321 answer = self.gateway_client.send_command(command)
-> 1322 return_value = get_return_value(
1323 answer, self.gateway_client, self.target_id, self.name)
1325 for temp_arg in temp_args:
1326 if hasattr(temp_arg, "_detach"):
File c:\Users\kunal\miniconda3\Lib\site-packages\pyspark\errors\exceptions\captured.py:169, in capture_sql_exception..deco(*a, **kw)
167 def deco(*a: Any, **kw: Any) -> Any:
168 try:
--> 169 return f(*a, **kw)
170 except Py4JJavaError as e:
171 converted = convert_exception(e.java_exception)
File c:\Users\kunal\miniconda3\Lib\site-packages\py4j\protocol.py:326, in get_return_value(answer, gateway_client, target_id, name)
324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
325 if answer[1] == REFERENCE_TYPE:
--> 326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n".
332 format(target_id, ".", name, value))
Py4JJavaError: An error occurred while calling o260.saveAsTable.
: java.lang.UnsatisfiedLinkError: o r g . a p a c h e . h a d o o p . i o . n a t i v e i o . N a t i v e I O $ W i n d o w s . a c c e s s 0 ( L j a v a / l a n g / S t r i n g ; I ) Z < b r / > a t o r g . a p a c h e . h a d o o p . i o . n a t i v e i o . N a t i v e I O $ W i n d o w s . a c c e s s 0 ( N a t i v e M e t h o d ) < b r / > a t o r g . a p a c h e . h a d o o p . i o . n a t i v e i o . N a t i v e I O $ W i n d o w s . a c c e s s ( N a t i v e I O . j a v a : 7 9 3 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . F i l e U t i l . c a n R e a d ( F i l e U t i l . j a v a : 1 2 4 9 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . F i l e U t i l . l i s t ( F i l e U t i l . j a v a : 1 4 5 4 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . R a w L o c a l F i l e S y s t e m . l i s t S t a t u s ( R a w L o c a l F i l e S y s t e m . j a v a : 6 0 1 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . F i l e S y s t e m . l i s t S t a t u s ( F i l e S y s t e m . j a v a : 1 9 7 2 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . F i l e S y s t e m . l i s t S t a t u s ( F i l e S y s t e m . j a v a : 2 0 1 4 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . C h e c k s u m F i l e S y s t e m . l i s t S t a t u s ( C h e c k s u m F i l e S y s t e m . j a v a : 7 6 1 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . F i l e S y s t e m . l i s t S t a t u s ( F i l e S y s t e m . j a v a : 1 9 7 2 ) < b r / > a t o r g . a p a c h e . h a d o o p . f s . F i l e S y s t e m . l i s t S t a t u s ( F i l e S y s t e m . j a v a : 2 0 1 4 ) < b r / > a t o r g . a p a c h e . h a d o o p . m a p r e d u c e . l i b . o u t p u t . F i l e O u t p u t C o m m i t t e r . g e t A l l C o m m i t t e d T a s k P a t h s ( F i l e O u t p u t C o m m i t t e r . j a v a : 3 3 4 ) < b r / > a t o r g . a p a c h e . h a d o o p . m a p r e d u c e . l i b . o u t p u t . F i l e O u t p u t C o m m i t t e r . c o m m i t J o b I n t e r n a l ( F i l e O u t p u t C o m m i t t e r . j a v a : 4 0 4 ) < b r / > a t o r g . a p a c h e . h a d o o p . m a p r e d u c e . l i b . o u t p u t . F i l e O u t p u t C o m m i t t e r . c o m m i t J o b ( F i l e O u t p u t C o m m i t t e r . j a v a : 3 7 7 ) < b r / > a t o r g . a p a c h e . p a r q u e t . h a d o o p . P a r q u e t O u t p u t C o m m i t t e r . c o m m i t J o b ( P a r q u e t O u t p u t C o m m i t t e r . j a v a : 4 8 ) < b r / > a t o r g . a p a c h e . s p a r k . i n t e r n a l . i o . H a d o o p M a p R e d u c e C o m m i t P r o t o c o l . c o m m i t J o b ( H a d o o p M a p R e d u c e C o m m i t P r o t o c o l . s c a l a : 1 9 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . d a t a s o u r c e s . F i l e F o r m a t W r i t e r $ . $ a n o n f u n $ w r i t e A n d C o m m i t $ 3 ( F i l e F o r m a t W r i t e r . s c a l a : 2 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . d a t a s o u r c e s . F i l e F o r m a t W r i t e r $ $ $ L a m b d a $ 3 5 9 3 / 9 1 2 4 7 6 9 6 3 . a p p l y $ m c V $ s p ( U n k n o w n S o u r c e ) < b r / > a t s c a l a . r u n t i m e . j a v a 8 . J F u n c t i o n 0 $ m c V $ s p . a p p l y ( J F u n c t i o n 0 $ m c V $ s p . j a v a : 2 3 ) < b r / > a t o r g . a p a c h e . s p a r k . u t i l . U t i l s $ . t i m e T a k e n M s ( U t i l s . s c a l a : 5 5 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . d a t a s o u r c e s . F i l e F o r m a t W r i t e r $ . w r i t e A n d C o m m i t ( F i l e F o r m a t W r i t e r . s c a l a : 2 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . d a t a s o u r c e s . F i l e F o r m a t W r i t e r $ . e x e c u t e W r i t e ( F i l e F o r m a t W r i t e r . s c a l a : 3 0 4 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . d a t a s o u r c e s . F i l e F o r m a t W r i t e r $ . w r i t e ( F i l e F o r m a t W r i t e r . s c a l a : 1 9 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . d a t a s o u r c e s . I n s e r t I n t o H a d o o p F s R e l a t i o n C o m m a n d . r u n ( I n s e r t I n t o H a d o o p F s R e l a t i o n C o m m a n d . s c a l a : 1 9 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . c o m m a n d . D a t a W r i t i n g C o m m a n d E x e c . s i d e E f f e c t R e s u l t $ l z y c o m p u t e ( c o m m a n d s . s c a l a : 1 1 3 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . c o m m a n d . D a t a W r i t i n g C o m m a n d E x e c . s i d e E f f e c t R e s u l t ( c o m m a n d s . s c a l a : 1 1 1 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . c o m m a n d . D a t a W r i t i n g C o m m a n d E x e c . e x e c u t e C o l l e c t ( c o m m a n d s . s c a l a : 1 2 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . Q u e r y E x e c u t i o n $ $ a n o n f u n $ e a g e r l y E x e c u t e C o m m a n d s $ 1 . $ a n o n f u n $ a p p l y O r E l s e $ 1 ( Q u e r y E x e c u t i o n . s c a l a : 1 0 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . Q u e r y E x e c u t i o n $ $ a n o n f u n $ e a g e r l y E x e c u t e C o m m a n d s $ 1 $ $ L a m b d a $ 2 9 5 6 / 2 3 4 7 0 5 3 5 4 . a p p l y ( U n k n o w n S o u r c e ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . S Q L E x e c u t i o n $ . $ a n o n f u n $ w i t h N e w E x e c u t i o n I d $ 6 ( S Q L E x e c u t i o n . s c a l a : 1 2 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . S Q L E x e c u t i o n $ $ $ L a m b d a $ 1 7 0 4 / 1 0 9 2 9 5 7 5 5 8 . a p p l y ( U n k n o w n S o u r c e ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . S Q L E x e c u t i o n $ . w i t h S Q L C o n f P r o p a g a t e d ( S Q L E x e c u t i o n . s c a l a : 2 0 1 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . S Q L E x e c u t i o n $ . $ a n o n f u n $ w i t h N e w E x e c u t i o n I d $ 1 ( S Q L E x e c u t i o n . s c a l a : 1 0 8 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . S Q L E x e c u t i o n $ $ $ L a m b d a $ 1 6 9 1 / 9 4 6 0 2 8 8 4 . a p p l y ( U n k n o w n S o u r c e ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . S p a r k S e s s i o n . w i t h A c t i v e ( S p a r k S e s s i o n . s c a l a : 9 0 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . S Q L E x e c u t i o n $ . w i t h N e w E x e c u t i o n I d ( S Q L E x e c u t i o n . s c a l a : 6 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . Q u e r y E x e c u t i o n $ $ a n o n f u n $ e a g e r l y E x e c u t e C o m m a n d s $ 1 . a p p l y O r E l s e ( Q u e r y E x e c u t i o n . s c a l a : 1 0 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . Q u e r y E x e c u t i o n $ $ a n o n f u n $ e a g e r l y E x e c u t e C o m m a n d s $ 1 . a p p l y O r E l s e ( Q u e r y E x e c u t i o n . s c a l a : 9 8 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . $ a n o n f u n $ t r a n s f o r m D o w n W i t h P r u n i n g $ 1 ( T r e e N o d e . s c a l a : 4 6 1 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e $ $ L a m b d a $ 1 2 9 1 / 9 0 5 5 6 5 3 2 9 . a p p l y ( U n k n o w n S o u r c e ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . C u r r e n t O r i g i n $ . w i t h O r i g i n ( o r i g i n . s c a l a : 7 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . t r a n s f o r m D o w n W i t h P r u n i n g ( T r e e N o d e . s c a l a : 4 6 1 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . o r g $ a p a c h e $ s p a r k $ s q l $ c a t a l y s t $ p l a n s $ l o g i c a l $ A n a l y s i s H e l p e r $ $ s u p e r $ t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g ( A n a l y s i s H e l p e r . s c a l a : 2 6 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g $ ( A n a l y s i s H e l p e r . s c a l a : 2 6 3 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . t r a n s f o r m D o w n ( T r e e N o d e . s c a l a : 4 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . Q u e r y E x e c u t i o n . e a g e r l y E x e c u t e C o m m a n d s ( Q u e r y E x e c u t i o n . s c a l a : 9 8 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . Q u e r y E x e c u t i o n . c o m m a n d E x e c u t e d $ l z y c o m p u t e ( Q u e r y E x e c u t i o n . s cala:85)
at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:83)
at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:142)
at org.apache.spark.sql.execution.datasources.DataSource.writeAndRead(DataSource.scala:511)
at org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand.saveDataIntoTable(createDataSourceTables.scala:229)
at org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand.run(createDataSourceTables.scala:183)
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:107)
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1$$Lambda$2956/234705354.apply(Unknown Source)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:125)
at org.apache.spark.sql.execution.SQLExecution$$$Lambda$1704/1092957558.apply(Unknown Source)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:201)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:108)
at org.apache.spark.sql.execution.SQLExecution$$$Lambda$1691/94602884.apply(Unknown Source)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:66)
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:107)
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:98)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:461)
at org.apache.spark.sql.catalyst.trees.TreeNode$$Lambda$1291/905565329.apply(Unknown Source)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:76)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:461)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:32)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:437)
at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:98)
at org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:85)
at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:83)
at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:142)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:859)
at org.apache.spark.sql.DataFrameWriter.createTable(DataFrameWriter.scala:700)
at org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:678)
at org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:571)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Unknown Source)
Я ожидал сохранить таблицу в каталоге как файл csv/parquet для дальнейшего использования, но создаются только папки, CSV не сохраняется.< /п>
Подробнее здесь: https://stackoverflow.com/questions/788 ... -dataframe
