From a5ef4b829e881b27d1e536420c3d7f0aabb9fd16 Mon Sep 17 00:00:00 2001 From: huzekang <1040080742@qq.com> Date: Thu, 27 Jun 2019 11:33:31 +0800 Subject: [PATCH] =?UTF-8?q?change:=201.=E6=9B=B4=E6=96=B0readme=202.sparks?= =?UTF-8?q?ql=E7=A4=BA=E4=BE=8B=E6=9B=B4=E6=96=B0=E4=BA=86=E5=86=99?= =?UTF-8?q?=E5=87=BA=E7=9A=84=E7=BC=96=E7=A0=81=E6=96=B9=E5=BC=8F=EF=BC=8C?= =?UTF-8?q?=E9=98=B2=E6=AD=A2=E5=9B=A0=E4=B8=BA=E7=B3=BB=E7=BB=9F=E7=8E=AF?= =?UTF-8?q?=E5=A2=83=E7=A1=AE=E5=AE=9Elib=E6=8A=A5=E9=94=99=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- .../com/wugui/sparkstarter/SparkHiveNewVersion.java | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cf151ef..b600e03 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ SparkSession spark = SparkSession 3.使用`mvn clean package`打包好的作业，并提交到本地安装好的spark环境上跑 ``` -~/opt/spark-2.4.0-bin-hadoop2.7 » bin/spark-submit --class "com.wugui.sparkstarter.SimpleApp" /Users/huzekang/study/spark-starter/target/spark-starter-1.0-SNAPSHOT.jar +~/opt/spark-2.4.0-bin-hadoop2.7 » bin/spark-submit --class "com.wugui.sparkstarter.SparkHiveNewVersion" /Users/huzekang/study/spark-starter/target/spark-starter-1.0-SNAPSHOT.jar ``` 4.打开spark server界面，可以看到已经完成的spark作业。 diff --git a/src/main/java/com/wugui/sparkstarter/SparkHiveNewVersion.java b/src/main/java/com/wugui/sparkstarter/SparkHiveNewVersion.java index 8c6b36c..e4e1a5a 100644 --- a/src/main/java/com/wugui/sparkstarter/SparkHiveNewVersion.java +++ b/src/main/java/com/wugui/sparkstarter/SparkHiveNewVersion.java @@ -15,14 +15,19 @@ public static void main(String[] args) { // 定义上下文 SparkSession spark = SparkSession .builder() - // 如果需要作业要以jar包形式提交到remote spark，则使用spark://host:port + // 如果需要作业要以jar包形式提交到remote spark，则使用spark://host:port // .master("spark://10.0.0.50:7077") - // 如果idea中测试则使用local。 - // 如果作业要以jar包形式提交到yarn则不设置master。 + + // 如果idea中测试则使用local。 + // 如果作业要以jar包形式提交到yarn则不设置master。 .master("local") + .appName("Java Spark SQL Starter ！！") .enableHiveSupport() - .config("spark.some.config.option", "some-value") + // 改变spark sql写出时使用的压缩编码。 + // 默认是snappy，可能会在用hive客户端查询时出现错误： + // Caused by: org.xerial.snappy.SnappyError: [FAILED_TO_LOAD_NATIVE_LIBRARY] null + .config("spark.sql.parquet.compression.codec", "gzip") .getOrCreate(); spark.sql("USE default");