From be638cf0a32ec1152be88b1a014b5508650cf8f7 Mon Sep 17 00:00:00 2001
From: huzekang <1040080742@qq.com>
Date: Fri, 6 Sep 2019 14:22:51 +0800
Subject: [PATCH] =?UTF-8?q?add:=201.scala=E8=AF=AD=E6=B3=95=E5=AD=A6?=
=?UTF-8?q?=E4=B9=A0=202.=20spark=20ml?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
spark-starter/pom.xml | 16 +-
.../wugui/sparkstarter/ml/SparkDouban.java | 18 +
.../com/wugui/sparkstarter/ml/SparkTFIDF.java | 63 +
.../sparkstarter/ml/TokenizerExample.java | 49 +
.../sparkstarter/ml/Word2VecExample.java | 53 +
.../src/main/resources/data/README.md | 3 +
.../src/main/resources/data/hot_movies.csv | 166 +
.../src/main/resources/data/user_movies.csv | 1007397 ++++++++++++++
.../resources/sample_movielens_ratings.txt | 1501 +
.../src/main/scala/AdvanceFunc.scala | 33 +
spark-starter/src/main/scala/BangSheng.scala | 21 +
spark-starter/src/main/scala/BreakTest.scala | 37 +
.../src/main/scala/CollectionTest.scala | 79 +
spark-starter/src/main/scala/FuncTest.scala | 32 +
spark-starter/src/main/scala/SparkPi.scala | 34 +
spark-starter/src/main/scala/TestVoMain.scala | 34 +
spark-starter/src/main/scala/UseJava.scala | 19 +
...70\347\224\250\345\207\275\346\225\260.md" | 385 +
18 files changed, 1009935 insertions(+), 5 deletions(-)
create mode 100644 spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkDouban.java
create mode 100644 spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkTFIDF.java
create mode 100644 spark-starter/src/main/java/com/wugui/sparkstarter/ml/TokenizerExample.java
create mode 100644 spark-starter/src/main/java/com/wugui/sparkstarter/ml/Word2VecExample.java
create mode 100644 spark-starter/src/main/resources/data/README.md
create mode 100755 spark-starter/src/main/resources/data/hot_movies.csv
create mode 100755 spark-starter/src/main/resources/data/user_movies.csv
create mode 100644 spark-starter/src/main/resources/sample_movielens_ratings.txt
create mode 100644 spark-starter/src/main/scala/AdvanceFunc.scala
create mode 100644 spark-starter/src/main/scala/BangSheng.scala
create mode 100644 spark-starter/src/main/scala/BreakTest.scala
create mode 100644 spark-starter/src/main/scala/CollectionTest.scala
create mode 100644 spark-starter/src/main/scala/FuncTest.scala
create mode 100644 spark-starter/src/main/scala/SparkPi.scala
create mode 100644 spark-starter/src/main/scala/TestVoMain.scala
create mode 100644 spark-starter/src/main/scala/UseJava.scala
create mode 100644 "spark-starter/src/main/scala/scala\346\225\260\346\215\256\347\273\223\346\236\204\345\222\214\345\270\270\347\224\250\345\207\275\346\225\260.md"
diff --git a/spark-starter/pom.xml b/spark-starter/pom.xml
index 2549d7b..283bd5f 100644
--- a/spark-starter/pom.xml
+++ b/spark-starter/pom.xml
@@ -51,11 +51,11 @@
-
-
-
-
-
+
+ org.apache.spark
+ spark-mllib_2.11
+ 2.4.0
+
@@ -151,6 +151,12 @@
+
+ org.scala-lang
+ scala-library
+ 2.11.12
+
+
diff --git a/spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkDouban.java b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkDouban.java
new file mode 100644
index 0000000..e2627c9
--- /dev/null
+++ b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkDouban.java
@@ -0,0 +1,18 @@
+package com.wugui.sparkstarter.ml;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+
+/**
+ * @program: bigdata-starter
+ * @author: huzekang
+ * @create: 2019-09-05 20:38
+ **/
+public class SparkDouban {
+ public static void main(String[] args) {
+ SparkSession sparkSession = SparkSession.builder().master("local").getOrCreate();
+ RDD rdd = sparkSession.read().text("/Users/huzekang/study/bigdata-starter/spark-starter/src/main/resources/data/hot_movies.csv").rdd();
+ }
+}
diff --git a/spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkTFIDF.java b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkTFIDF.java
new file mode 100644
index 0000000..327fb97
--- /dev/null
+++ b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/SparkTFIDF.java
@@ -0,0 +1,63 @@
+package com.wugui.sparkstarter.ml;
+
+import org.apache.spark.SparkContext;
+import org.apache.spark.ml.feature.HashingTF;
+import org.apache.spark.ml.feature.IDF;
+import org.apache.spark.ml.feature.IDFModel;
+import org.apache.spark.ml.feature.Tokenizer;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @program: bigdata-starter
+ * @author: huzekang
+ * @create: 2019-09-05 18:49
+ **/
+public class SparkTFIDF {
+ public static void main(String[] args) {
+ SparkSession spark = SparkSession
+ .builder()
+ .appName("TFIDF")
+ .master("local")
+ .getOrCreate();
+
+ List data = Arrays.asList(
+ RowFactory.create(0.0, "Hi I heard about Spark"),
+ RowFactory.create(0.0, "I wish Java could use case classes"),
+ RowFactory.create(1.0, "Logistic regression models are neat")
+ );
+
+ StructType schema = new StructType(new StructField[]{
+ new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
+ new StructField("sentence", DataTypes.StringType, false, Metadata.empty())
+ });
+ Dataset sentenceData = spark.createDataFrame(data, schema);
+
+ Tokenizer tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words");
+ Dataset wordsData = tokenizer.transform(sentenceData);
+
+ int numFeatures = 20;
+ HashingTF hashingTF = new HashingTF();
+ hashingTF.setInputCol("words")
+ .setOutputCol("rawFeatures");
+ //.setNumFeatures(numFeatures);
+
+ Dataset featurizedData = hashingTF.transform(wordsData);
+ featurizedData.show();
+
+ IDF idf = new IDF().setInputCol("rawFeatures").setOutputCol("features");
+ IDFModel idfModel = idf.fit(featurizedData);
+ Dataset rescalsedData = idfModel.transform(featurizedData);
+
+ System.out.println();
+ }
+}
diff --git a/spark-starter/src/main/java/com/wugui/sparkstarter/ml/TokenizerExample.java b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/TokenizerExample.java
new file mode 100644
index 0000000..85411f7
--- /dev/null
+++ b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/TokenizerExample.java
@@ -0,0 +1,49 @@
+package com.wugui.sparkstarter.ml;
+
+import org.apache.spark.ml.feature.Tokenizer;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+import java.util.List;
+
+public class TokenizerExample
+{
+ public static void main(String[] args)
+ {
+ SparkSession spark = SparkSession
+ .builder()
+ .appName("Tokenizer")
+ .master("local")
+ .getOrCreate();
+
+ List data = java.util.Arrays.asList(
+ RowFactory.create(0, "Hi I heard about spark"),
+ RowFactory.create(1, "I wish Java could use case classes"),
+ RowFactory.create(2, "Logistic,regression,models,are,neat")
+ );
+
+ StructType schema = new StructType(new StructField[]{
+ new StructField("label", DataTypes.IntegerType, false, Metadata.empty()),
+ new StructField("sentence",DataTypes.StringType, false, Metadata.empty())
+ });
+
+ Dataset sentenceDataFrame = spark.createDataFrame(data, schema);
+
+ Tokenizer tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words");
+
+ Dataset wordsData = tokenizer.transform(sentenceDataFrame);
+ for (Row r : wordsData.select("words","label").takeAsList(3))
+ {
+ List words = r.getList(0);
+ for (String word : words) {
+ System.out.println(word);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/spark-starter/src/main/java/com/wugui/sparkstarter/ml/Word2VecExample.java b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/Word2VecExample.java
new file mode 100644
index 0000000..39b36ba
--- /dev/null
+++ b/spark-starter/src/main/java/com/wugui/sparkstarter/ml/Word2VecExample.java
@@ -0,0 +1,53 @@
+package com.wugui.sparkstarter.ml;
+
+import org.apache.spark.ml.feature.Word2Vec;
+import org.apache.spark.ml.feature.Word2VecModel;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.*;
+
+import java.util.Arrays;
+import java.util.List;
+
+
+public class Word2VecExample
+{
+ public static void main(String[] args)
+ {
+ SparkSession spark = SparkSession
+ .builder()
+ .appName("TFIDF")
+ .master("local")
+ .getOrCreate();
+
+ List data = Arrays.asList(
+ RowFactory.create(Arrays.asList("Hi I herd about spark".split(" "))),
+ RowFactory.create(Arrays.asList("I wish Java could use case classes".split(" "))),
+ RowFactory.create(Arrays.asList("Logistic regression models are neat".split(" ")))
+ );
+
+ StructType schema = new StructType(new StructField[]{
+ new StructField("text", new ArrayType(DataTypes.StringType, true), false, Metadata.empty())
+ });
+
+ Dataset documentDF = spark.createDataFrame(data, schema);
+
+ Word2Vec word2Vec = new Word2Vec()
+ .setInputCol("text")
+ .setOutputCol("result")
+ .setVectorSize(7)
+ .setMinCount(0);
+
+ Word2VecModel model = word2Vec.fit(documentDF);
+ Dataset|