add: 1.clickhouse支持

Kyofin · Sep 30, 2019 · 617ea63 · 617ea63
1 parent 53de0fb
commit 617ea63
Show file tree

Hide file tree

Showing 2 changed files with 102 additions and 1 deletion.
diff --git a/spark-starter/pom.xml b/spark-starter/pom.xml
@@ -17,13 +17,33 @@
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.11</artifactId>
             <version>2.4.0</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jersey-server</artifactId>
+                    <groupId>org.glassfish.jersey.core</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <!--不加上会出现org/glassfish/jersey/server/spi/Container not found-->
         <dependency>
             <groupId>org.glassfish.jersey.core</groupId>
             <artifactId>jersey-server</artifactId>
             <version>2.0-m03</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jersey-common</artifactId>
+                    <groupId>org.glassfish.jersey.core</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -64,7 +84,19 @@
             <version>5.5.1</version>
         </dependency>
 
-
+        <dependency>
+            <groupId>oracle.jdbc</groupId>
+            <artifactId>ojdbc</artifactId>
+            <version>6</version>
+            <scope>system</scope>
+            <systemPath>${project.basedir}/src/main/resources/lib/ojdbc-6.jar</systemPath>
+        </dependency>
+        <!--clickhouse-->
+        <dependency>
+            <groupId>ru.yandex.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.2</version>
+        </dependency>
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
@@ -89,6 +121,10 @@
                     <artifactId>hadoop-common</artifactId>
                     <groupId>org.apache.hadoop</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -109,12 +145,22 @@
                     <artifactId>disruptor</artifactId>
                     <groupId>com.lmax</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
             <version>2.6.0</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
@@ -125,6 +171,10 @@
                     <artifactId>hadoop-common</artifactId>
                     <groupId>org.apache.hadoop</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -139,6 +189,12 @@
             <groupId>org.apache.phoenix</groupId>
             <artifactId>phoenix-core</artifactId>
             <version>4.14.0-HBase-1.2</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

diff --git a/spark-starter/src/main/java/com/wugui/sparkstarter/ClickHouseTest.java b/spark-starter/src/main/java/com/wugui/sparkstarter/ClickHouseTest.java
@@ -0,0 +1,45 @@
+package com.wugui.sparkstarter;
+
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.SparkSession;
+
+import java.util.Properties;
+
+/**
+ * @program: bigdata-starter
+ * @author: huzekang
+ * @create: 2019-09-29 15:30
+ **/
+public class ClickHouseTest {
+    public static void main(String[] args) {
+        SparkSession sparkSession = SparkSession
+                .builder()
+                .appName("clickhouse")
+                .master("local")
+                .getOrCreate();
+
+        Dataset<Row> df = sparkSession.read()
+                .format("jdbc")
+                .option("user", "yibo_test")
+                .option("password", "yibo123")
+                .option("driver", "oracle.jdbc.driver.OracleDriver")
+                .option("url", "jdbc:oracle:thin:@192.168.1.130:1521:orcl")
+                .option("dbtable", "TB_LIS_INDICATORS")
+                .load();
+
+
+        Properties connectionProperties = new Properties();
+        connectionProperties.put("driver", "ru.yandex.clickhouse.ClickHouseDriver");
+        connectionProperties.put("batchsize", "50000");
+        // 设置事务
+        connectionProperties.put("isolationLevel", "NONE");
+        // 设置并发
+        connectionProperties.put("numPartitions", "5");
+
+        df.write()
+                .mode(SaveMode.Append)
+                .jdbc("jdbc:clickhouse://192.168.1.39:8123", "TB_LIS_INDICATORS", connectionProperties);
+    }
+}