1119 39

Artificial Intelligence for Big Data [推广有奖]

0%

-

TA的文库  其他...

Python Programming

SAS Programming

Structural Equation Modeling

16

12314669 个

2816 点

2797 点

2624 点

429616 点

18429

80

6971 小时

2005-4-23

2018-10-16

Nicolle   发表于 2018-6-13 04:12:48 |只看作者 |倒序
1论坛币

本帖被以下文库推荐

Nicolle   发表于 2018-6-13 04:14:10 |只看作者
 import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.linalg.Vectors import org.apache.spark.ml.regression.LinearRegression val linearRegrsssionSampleData = sc.textFile("aibd/linear_regression_sample.txt") val labeledData = linearRegrsssionSampleData.map { line =>   val parts = line.split(',')   LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).toDouble)) }.cache().toDF val lr = new LinearRegression() val model = lr.fit(labeledData) val summary = model.summary println("R-squared = "+ summary.r2)复制代码

Nicolle   发表于 2018-6-13 04:14:57 |只看作者
 import org.apache.spark.ml.classification.LogisticRegression // Load training data val training = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") val lr = new LogisticRegression()   .setMaxIter(10)   .setRegParam(0.3)   .setElasticNetParam(0.8) // Fit the model val lrModel = lr.fit(training) // Print the coefficients and intercept for logistic regression println(s"Coefficients: ${lrModel.coefficients} Intercept:${lrModel.intercept}") // We can also use the multinomial family for binary classification val mlr = new LogisticRegression()   .setMaxIter(10)   .setRegParam(0.3)   .setElasticNetParam(0.8)   .setFamily("multinomial") val mlrModel = mlr.fit(training) // Print the coefficients and intercepts for logistic regression with multinomial family println(s"Multinomial coefficients: ${mlrModel.coefficientMatrix}") println(s"Multinomial intercepts:${mlrModel.interceptVector}")复制代码

Nicolle   发表于 2018-6-13 04:15:26 |只看作者
 import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.linalg.Vectors import org.apache.spark.ml.clustering.KMeans val kmeansSampleData = sc.textFile("aibd/k-means-sample.txt") val labeledData = kmeansSampleData.map { line =>   val parts = line.split(',')   LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).toDouble, parts(2).toDouble)) }.cache().toDF val kmeans = new KMeans() .setK(2) // default value is 2 .setFeaturesCol("features") .setMaxIter(3) // default Max Iteration is 20 .setPredictionCol("prediction") .setSeed(1L) val model = kmeans.fit(labeledData) summary.predictions.show model.clusterCenters.foreach(println)复制代码

Nicolle   发表于 2018-6-13 04:16:11 |只看作者
 import org.apache.spark.mllib.linalg.Matrix import org.apache.spark.mllib.linalg.Vectors import org.apache.spark.mllib.linalg.distributed.RowMatrix val data = Array(Vectors.dense(2.0, 1.0, 75.0, 18.0, 1.0,2), Vectors.dense(0.0, 1.0, 21.0, 28.0, 2.0,4), Vectors.dense(0.0, 1.0, 32.0, 61.0, 5.0,10), Vectors.dense(0.0, 1.0, 56.0, 39.0, 2.0,4), Vectors.dense(1.0, 1.0, 73.0, 81.0, 3.0,6), Vectors.dense(0.0, 1.0, 97.0, 59.0, 7.0,14)) val rows = sc.parallelize(data) val mat: RowMatrix = new RowMatrix(rows) // Principal components are stored in a local dense matrix. val pc: Matrix = mat.computePrincipalComponents(2) // Project the rows to the linear space spanned by the top 2 principal components. val projected: RowMatrix = mat.multiply(pc) projected.rows.foreach(println)复制代码

Nicolle   发表于 2018-6-13 04:16:48 |只看作者
 import org.apache.spark.mllib.linalg.Matrix import org.apache.spark.mllib.linalg.Vectors import org.apache.spark.mllib.linalg.Vector import org.apache.spark.mllib.linalg.distributed.RowMatrix import org.apache.spark.mllib.linalg.SingularValueDecomposition val data = Array(Vectors.dense(2.0, 1.0, 75.0, 18.0, 1.0,2), Vectors.dense(0.0, 1.0, 21.0, 28.0, 2.0,4), Vectors.dense(0.0, 1.0, 32.0, 61.0, 5.0,10), Vectors.dense(0.0, 1.0, 56.0, 39.0, 2.0,4), Vectors.dense(1.0, 1.0, 73.0, 81.0, 3.0,6), Vectors.dense(0.0, 1.0, 97.0, 59.0, 7.0,14)) val rows = sc.parallelize(data) val mat: RowMatrix = new RowMatrix(rows) val svd: SingularValueDecomposition[RowMatrix, Matrix] = mat.computeSVD(3, computeU = true) val U: RowMatrix = svd.U // The U factor is stored as a row matrix val s: Vector = svd.s         // The sigma factor is stored as a singular vector val V: Matrix = svd.V         // The V factor is stored as a local dense matrix复制代码

7
Nicolle   发表于 2018-6-13 04:17:27 |只看作者
 object FeedForwardNetworkWithSpark {         def main(args:Array[String]): Unit ={         val recordReader:RecordReader = new CSVRecordReader(0,",")         val conf = new SparkConf()         .setMaster("spark://master:7077")         .setAppName("FeedForwardNetwork-Iris")         val sc = new SparkContext(conf)         val numInputs:Int = 4         val outputNum = 3         val iterations =1         val multiLayerConfig:MultiLayerConfiguration = new         NeuralNetConfiguration.Builder()                 .seed(12345)                 .iterations(iterations)                 .optimizationAlgo(OptimizationAlgorithm                 .STOCHASTIC_GRADIENT_DESCENT)                 .learningRate(1e-1)                 .l1(0.01).regularization(true).l2(1e-3)                 .list(3)                 .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(3)                 .activation("tanh")                 .weightInit(WeightInit.XAVIER)                 .build())                 .layer(1, new DenseLayer.Builder().nIn(3).nOut(2)                 .activation("tanh")                 .weightInit(WeightInit.XAVIER)                 .build())                 .layer(2, new                 OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)                 .weightInit(WeightInit.XAVIER)                 .activation("softmax")                 .nIn(2).nOut(outputNum).build())                 .backprop(true).pretrain(false)                 .build                 val network:MultiLayerNetwork = new        MultiLayerNetwork(multiLayerConfig)                 network.init                 network.setUpdater(null)                 val sparkNetwork:SparkDl4jMultiLayer = new                 SparkDl4jMultiLayer(sc,network)                 val nEpochs:Int = 6                 val listBuffer = new ListBuffer[Array[Float]]()                 (0 until nEpochs).foreach{i =>                 val net:MultiLayerNetwork =                 sparkNetwork.fit("file:////                 iris_shuffled_normalized_csv.txt",4,recordReader)                 listBuffer +=(net.params.data.asFloat().clone())                 }                 println("Parameters vs. iteration Output: ")                 (0 until listBuffer.size).foreach{i =>                 println(i+"\t"+listBuffer(i).mkString)}                 }         }复制代码

8
Nicolle   发表于 2018-6-13 04:18:19 |只看作者
 import org.apache.spark.mllib.linalg.Matrix import org.apache.spark.mllib.linalg.Vectors import org.apache.spark.mllib.linalg.distributed.RowMatrix import scala.util.Random import org.apache.spark.mllib.clustering._ import org.apache.spark.ml.clustering._ import org.apache.spark.mllib.clustering.KMeans import org.apache.spark.mllib.clustering.FuzzyCMeans import org.apache.spark.mllib.clustering.FuzzyCMeans._ import org.apache.spark.mllib.clustering.FuzzyCMeansModel val points = Seq(       Vectors.dense(0.0, 0.0),       Vectors.dense(0.0, 0.1),       Vectors.dense(0.1, 0.0),       Vectors.dense(9.0, 0.0),       Vectors.dense(9.0, 0.2),       Vectors.dense(9.2, 0.0)     )     val rdd = sc.parallelize(points, 3).cache()     for (initMode <- Seq(KMeans.RANDOM, KMeans.K_MEANS_PARALLEL)) {       (1 to 10).map(_ * 2) foreach { fuzzifier =>         val model = org.apache.spark.mllib.clustering.FuzzyCMeans.train(rdd, k = 2, maxIterations = 10, runs = 10, initMode,           seed = 26031979L, m = fuzzifier)         val fuzzyPredicts = model.fuzzyPredict(rdd).collect()                  rdd.collect() zip fuzzyPredicts foreach { fuzzyPredict =>           println(s" Point ${fuzzyPredict._1}") fuzzyPredict._2 foreach{clusterAndProbability => println(s"Probability to belong to cluster${clusterAndProbability._1} " +               s"is \${"%.2f".format(clusterAndProbability._2)}")           }         }       }     }         复制代码

9
edmcheng 发表于 2018-6-13 05:51:44 |只看作者
 Thanks

10
jgw1213 发表于 2018-6-13 06:13:52 |只看作者
 Thanks

 您需要登录后才可以回帖 登录 | 我要注册 回帖后跳转到最后一页