[Scala Study]Linear Regression using Scala

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

1. // scalastyle:off println
   
2. package org.apache.spark.examples.ml
   
3. 
   
4. import scala.collection.mutable
   
5. import scala.language.reflectiveCalls
   
6. 
   
7. import scopt.OptionParser
   
8. 
   
9. import org.apache.spark.{SparkConf, SparkContext}
   
10. import org.apache.spark.examples.mllib.AbstractParams
    
11. import org.apache.spark.ml.{Pipeline, PipelineStage}
    
12. import org.apache.spark.ml.regression.{LinearRegression, LinearRegressionModel}
    
13. import org.apache.spark.sql.DataFrame
    
14. 
    
15. /**
    
16. * An example runner for linear regression with elastic-net (mixing L1/L2) regularization.
    
17. * Run with
    
18. * {{{
    
19. * bin/run-example ml.LinearRegressionExample [options]
    
20. * }}}
    
21. * A synthetic dataset can be found at `data/mllib/sample_linear_regression_data.txt` which can be
    
22. * trained by
    
23. * {{{
    
24. * bin/run-example ml.LinearRegressionExample --regParam 0.15 --elasticNetParam 1.0 \
    
25. *   data/mllib/sample_linear_regression_data.txt
    
26. * }}}
    
27. * If you use it as a template to create your own app, please use `spark-submit` to submit your app.
    
28. */
    
29. object LinearRegressionExample {
    
30. 
    
31.   case class Params(
    
32.       input: String = null,
    
33.       testInput: String = "",
    
34.       dataFormat: String = "libsvm",
    
35.       regParam: Double = 0.0,
    
36.       elasticNetParam: Double = 0.0,
    
37.       maxIter: Int = 100,
    
38.       tol: Double = 1E-6,
    
39.       fracTest: Double = 0.2) extends AbstractParams[Params]
    
40. 
    
41.   def main(args: Array[String]) {
    
42.     val defaultParams = Params()
    
43. 
    
44.     val parser = new OptionParser[Params]("LinearRegressionExample") {
    
45.       head("LinearRegressionExample: an example Linear Regression with Elastic-Net app.")
    
46.       opt[Double]("regParam")
    
47.         .text(s"regularization parameter, default: ${defaultParams.regParam}")
    
48.         .action((x, c) => c.copy(regParam = x))
    
49.       opt[Double]("elasticNetParam")
    
50.         .text(s"ElasticNet mixing parameter. For alpha = 0, the penalty is an L2 penalty. " +
    
51.         s"For alpha = 1, it is an L1 penalty. For 0 < alpha < 1, the penalty is a combination of " +
    
52.         s"L1 and L2, default: ${defaultParams.elasticNetParam}")
    
53.         .action((x, c) => c.copy(elasticNetParam = x))
    
54.       opt[Int]("maxIter")
    
55.         .text(s"maximum number of iterations, default: ${defaultParams.maxIter}")
    
56.         .action((x, c) => c.copy(maxIter = x))
    
57.       opt[Double]("tol")
    
58.         .text(s"the convergence tolerance of iterations, Smaller value will lead " +
    
59.         s"to higher accuracy with the cost of more iterations, default: ${defaultParams.tol}")
    
60.         .action((x, c) => c.copy(tol = x))
    
61.       opt[Double]("fracTest")
    
62.         .text(s"fraction of data to hold out for testing.  If given option testInput, " +
    
63.         s"this option is ignored. default: ${defaultParams.fracTest}")
    
64.         .action((x, c) => c.copy(fracTest = x))
    
65.       opt[String]("testInput")
    
66.         .text(s"input path to test dataset.  If given, option fracTest is ignored." +
    
67.         s" default: ${defaultParams.testInput}")
    
68.         .action((x, c) => c.copy(testInput = x))
    
69.       opt[String]("dataFormat")
    
70.         .text("data format: libsvm (default), dense (deprecated in Spark v1.1)")
    
71.         .action((x, c) => c.copy(dataFormat = x))
    
72.       arg[String]("<input>")
    
73.         .text("input path to labeled examples")
    
74.         .required()
    
75.         .action((x, c) => c.copy(input = x))
    
76.       checkConfig { params =>
    
77.         if (params.fracTest < 0 || params.fracTest >= 1) {
    
78.           failure(s"fracTest ${params.fracTest} value incorrect; should be in [0,1).")
    
79.         } else {
    
80.           success
    
81.         }
    
82.       }
    
83.     }
    
84. 
    
85.     parser.parse(args, defaultParams).map { params =>
    
86.       run(params)
    
87.     }.getOrElse {
    
88.       sys.exit(1)
    
89.     }
    
90.   }
    
91. 
    
92.   def run(params: Params) {
    
93.     val conf = new SparkConf().setAppName(s"LinearRegressionExample with $params")
    
94.     val sc = new SparkContext(conf)
    
95. 
    
96.     println(s"LinearRegressionExample with parameters:\n$params")
    
97. 
    
98.     // Load training and test data and cache it.
    
99.     val (training: DataFrame, test: DataFrame) = DecisionTreeExample.loadDatasets(sc, params.input,
    
100.       params.dataFormat, params.testInput, "regression", params.fracTest)
     
101. 
     
102.     val lir = new LinearRegression()
     
103.       .setFeaturesCol("features")
     
104.       .setLabelCol("label")
     
105.       .setRegParam(params.regParam)
     
106.       .setElasticNetParam(params.elasticNetParam)
     
107.       .setMaxIter(params.maxIter)
     
108.       .setTol(params.tol)
     
109. 
     
110.     // Train the model
     
111.     val startTime = System.nanoTime()
     
112.     val lirModel = lir.fit(training)
     
113.     val elapsedTime = (System.nanoTime() - startTime) / 1e9
     
114.     println(s"Training time: $elapsedTime seconds")
     
115. 
     
116.     // Print the weights and intercept for linear regression.
     
117.     println(s"Weights: ${lirModel.coefficients} Intercept: ${lirModel.intercept}")
     
118. 
     
119.     println("Training data results:")
     
120.     DecisionTreeExample.evaluateRegressionModel(lirModel, training, "label")
     
121.     println("Test data results:")
     
122.     DecisionTreeExample.evaluateRegressionModel(lirModel, test, "label")
     
123. 
     
124.     sc.stop()
     
125.   }
     
126. }
     
127. // scalastyle:on println

复制代码

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏0 回帖

关键词：regression regressio regress Linear SCALA example import