[Case Study]Simple Text Classification using Java

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

1. /
   
2. 
   
3. package org.apache.spark.examples.ml;
   
4. 
   
5. import java.util.List;
   
6. 
   
7. import com.google.common.collect.Lists;
   
8. 
   
9. import org.apache.spark.SparkConf;
   
10. import org.apache.spark.api.java.JavaSparkContext;
    
11. import org.apache.spark.ml.Pipeline;
    
12. import org.apache.spark.ml.PipelineModel;
    
13. import org.apache.spark.ml.PipelineStage;
    
14. import org.apache.spark.ml.classification.LogisticRegression;
    
15. import org.apache.spark.ml.feature.HashingTF;
    
16. import org.apache.spark.ml.feature.Tokenizer;
    
17. import org.apache.spark.sql.DataFrame;
    
18. import org.apache.spark.sql.Row;
    
19. import org.apache.spark.sql.SQLContext;
    
20. 
    
21. /**
    
22. * A simple text classification pipeline that recognizes "spark" from input text. It uses the Java
    
23. * bean classes {@link LabeledDocument} and {@link Document} defined in the Scala counterpart of
    
24. * this example {@link SimpleTextClassificationPipeline}. Run with
    
25. * <pre>
    
26. * bin/run-example ml.JavaSimpleTextClassificationPipeline
    
27. * </pre>
    
28. */
    
29. public class JavaSimpleTextClassificationPipeline {
    
30. 
    
31.   public static void main(String[] args) {
    
32.     SparkConf conf = new SparkConf().setAppName("JavaSimpleTextClassificationPipeline");
    
33.     JavaSparkContext jsc = new JavaSparkContext(conf);
    
34.     SQLContext jsql = new SQLContext(jsc);
    
35. 
    
36.     // Prepare training documents, which are labeled.
    
37.     List<LabeledDocument> localTraining = Lists.newArrayList(
    
38.       new LabeledDocument(0L, "a b c d e spark", 1.0),
    
39.       new LabeledDocument(1L, "b d", 0.0),
    
40.       new LabeledDocument(2L, "spark f g h", 1.0),
    
41.       new LabeledDocument(3L, "hadoop mapreduce", 0.0));
    
42.     DataFrame training = jsql.createDataFrame(jsc.parallelize(localTraining), LabeledDocument.class);
    
43. 
    
44.     // Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
    
45.     Tokenizer tokenizer = new Tokenizer()
    
46.       .setInputCol("text")
    
47.       .setOutputCol("words");
    
48.     HashingTF hashingTF = new HashingTF()
    
49.       .setNumFeatures(1000)
    
50.       .setInputCol(tokenizer.getOutputCol())
    
51.       .setOutputCol("features");
    
52.     LogisticRegression lr = new LogisticRegression()
    
53.       .setMaxIter(10)
    
54.       .setRegParam(0.001);
    
55.     Pipeline pipeline = new Pipeline()
    
56.       .setStages(new PipelineStage[] {tokenizer, hashingTF, lr});
    
57. 
    
58.     // Fit the pipeline to training documents.
    
59.     PipelineModel model = pipeline.fit(training);
    
60. 
    
61.     // Prepare test documents, which are unlabeled.
    
62.     List<Document> localTest = Lists.newArrayList(
    
63.       new Document(4L, "spark i j k"),
    
64.       new Document(5L, "l m n"),
    
65.       new Document(6L, "spark hadoop spark"),
    
66.       new Document(7L, "apache hadoop"));
    
67.     DataFrame test = jsql.createDataFrame(jsc.parallelize(localTest), Document.class);
    
68. 
    
69.     // Make predictions on test documents.
    
70.     DataFrame predictions = model.transform(test);
    
71.     for (Row r: predictions.select("id", "text", "probability", "prediction").collect()) {
    
72.       System.out.println("(" + r.get(0) + ", " + r.get(1) + ") --> prob=" + r.get(2)
    
73.           + ", prediction=" + r.get(3));
    
74.     }
    
75. 
    
76.     jsc.stop();
    
77.   }
    
78. }

复制代码

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏0 回帖

关键词：Case study cation simple study Using package import Java