From b0f4231ea669c093635a8881258f283bc7676069 Mon Sep 17 00:00:00 2001 From: huzekang <1040080742@qq.com> Date: Wed, 4 Sep 2019 09:50:44 +0800 Subject: [PATCH] =?UTF-8?q?change:1.=E6=9B=B4=E6=96=B0hbase=E7=9B=B8?= =?UTF-8?q?=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spark-starter/pom.xml | 12 +++++++ .../sparkstarter/hbase/SparkHbasePhoenix.java | 11 +++--- .../src/main/resources/hbase-site.xml | 35 +++++++++++++++++++ 3 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 spark-starter/src/main/resources/hbase-site.xml diff --git a/spark-starter/pom.xml b/spark-starter/pom.xml index 92b979d..2549d7b 100644 --- a/spark-starter/pom.xml +++ b/spark-starter/pom.xml @@ -127,6 +127,12 @@ + + org.apache.phoenix + phoenix-spark + 4.14.0-HBase-1.2 + provided + @@ -134,6 +140,12 @@ phoenix-core 4.14.0-HBase-1.2 + + + org.apache.phoenix + phoenix-client + 4.14.1-HBase-1.2 + diff --git a/spark-starter/src/main/java/com/wugui/sparkstarter/hbase/SparkHbasePhoenix.java b/spark-starter/src/main/java/com/wugui/sparkstarter/hbase/SparkHbasePhoenix.java index 2e4ab6a..c36ee46 100644 --- a/spark-starter/src/main/java/com/wugui/sparkstarter/hbase/SparkHbasePhoenix.java +++ b/spark-starter/src/main/java/com/wugui/sparkstarter/hbase/SparkHbasePhoenix.java @@ -1,8 +1,6 @@ package com.wugui.sparkstarter.hbase; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.*; /** * @program: bigdata-starter @@ -10,7 +8,7 @@ * @create: 2019-08-30 18:07 **/ public class SparkHbasePhoenix { - public static void main(String[] args) { + public static void main(String[] args) throws AnalysisException { SparkSession sparkSession = SparkSession.builder().appName("SparkHBaseDataFrame").master("local").getOrCreate(); Dataset dataset = sparkSession.read() @@ -18,8 +16,11 @@ public static void main(String[] args) { .option("driver", "org.apache.phoenix.jdbc.PhoenixDriver") .option("phoenix.schema.isNamespaceMappingEnabled", "true") .option("url", "jdbc:phoenix:cdh01:2181") - .option("dbtable", "userInfo") + .option("dbtable", "patient_test") .load(); dataset.printSchema(); + dataset.select("ID").show(); + dataset.createTempView("abc"); + sparkSession.sql("select empi,id from abc where id> 1005 ").show(); } } diff --git a/spark-starter/src/main/resources/hbase-site.xml b/spark-starter/src/main/resources/hbase-site.xml new file mode 100644 index 0000000..4bb2b9b --- /dev/null +++ b/spark-starter/src/main/resources/hbase-site.xml @@ -0,0 +1,35 @@ + + + + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + phoenix.schema.isNamespaceMappingEnabled + true + + + phoenix.schema.mapSystemTablesToNamespace + true + +