[Case Study]Term Frequency-Inverse Document Frequency using Java

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

1. */
   
2. 
   
3. package org.apache.spark.examples.ml;
   
4. 
   
5. // $example on$
   
6. import java.util.Arrays;
   
7. 
   
8. import org.apache.spark.SparkConf;
   
9. import org.apache.spark.api.java.JavaRDD;
   
10. import org.apache.spark.api.java.JavaSparkContext;
    
11. import org.apache.spark.ml.feature.HashingTF;
    
12. import org.apache.spark.ml.feature.IDF;
    
13. import org.apache.spark.ml.feature.IDFModel;
    
14. import org.apache.spark.ml.feature.Tokenizer;
    
15. import org.apache.spark.mllib.linalg.Vector;
    
16. import org.apache.spark.sql.DataFrame;
    
17. import org.apache.spark.sql.Row;
    
18. import org.apache.spark.sql.RowFactory;
    
19. import org.apache.spark.sql.SQLContext;
    
20. import org.apache.spark.sql.types.DataTypes;
    
21. import org.apache.spark.sql.types.Metadata;
    
22. import org.apache.spark.sql.types.StructField;
    
23. import org.apache.spark.sql.types.StructType;
    
24. // $example off$
    
25. 
    
26. public class JavaTfIdfExample {
    
27.   public static void main(String[] args) {
    
28.     SparkConf conf = new SparkConf().setAppName("JavaTfIdfExample");
    
29.     JavaSparkContext jsc = new JavaSparkContext(conf);
    
30.     SQLContext sqlContext = new SQLContext(jsc);
    
31. 
    
32.     // $example on$
    
33.     JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
    
34.       RowFactory.create(0, "Hi I heard about Spark"),
    
35.       RowFactory.create(0, "I wish Java could use case classes"),
    
36.       RowFactory.create(1, "Logistic regression models are neat")
    
37.     ));
    
38.     StructType schema = new StructType(new StructField[]{
    
39.       new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
    
40.       new StructField("sentence", DataTypes.StringType, false, Metadata.empty())
    
41.     });
    
42.     DataFrame sentenceData = sqlContext.createDataFrame(jrdd, schema);
    
43.     Tokenizer tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words");
    
44.     DataFrame wordsData = tokenizer.transform(sentenceData);
    
45.     int numFeatures = 20;
    
46.     HashingTF hashingTF = new HashingTF()
    
47.       .setInputCol("words")
    
48.       .setOutputCol("rawFeatures")
    
49.       .setNumFeatures(numFeatures);
    
50.     DataFrame featurizedData = hashingTF.transform(wordsData);
    
51.     IDF idf = new IDF().setInputCol("rawFeatures").setOutputCol("features");
    
52.     IDFModel idfModel = idf.fit(featurizedData);
    
53.     DataFrame rescaledData = idfModel.transform(featurizedData);
    
54.     for (Row r : rescaledData.select("features", "label").take(3)) {
    
55.       Vector features = r.getAs(0);
    
56.       Double label = r.getDouble(1);
    
57.       System.out.println(features);
    
58.       System.out.println(label);
    
59.     }
    
60.     // $example off$
    
61. 
    
62.     jsc.stop();
    
63.   }
    
64. }

复制代码

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏1 回帖

关键词：Case study Frequency Document inverse DOCUME example package import Java