WeatherDataAnalysis.json

{"paragraphs":[{"text":"import org.apache.spark.SparkContext\nimport org.apache.spark.SparkContext._\nimport org.apache.spark.SparkConf\nimport java.net.URL\nimport java.nio.charset.Charset\nimport org.apache.commons.io.IOUtils\nimport org.apache.spark.sql.SparkSession\nimport spark.implicits._\n\n      val conf = new SparkConf().setAppName(\"Simple Application\").setMaster(\"local[*]\")\n      conf.set(\"spark.driver.allowMultipleContexts\", \"true\")\n      val sc = new SparkContext(conf)\n      val weather = sc.textFile(\"/Users/Parth/Downloads/2015.csv\")\n    //   println(weather.first())\n      val weatherParse = weather.map(line => line.split(\",\"))\n    //   println(weatherParse.first()(0))\n      val weatherPrecp = weatherParse.filter(x => x(2) == \"PRCP\")\n    //   println(weatherPrecp.first()(0))\n      val weatherPrecpCountByKey = weatherPrecp.map( x => (x(0), (x(3).toInt, 1)))\n    //   println(weatherPrecpCountByKey.first())\n      val weatherPrecpAddByKey = weatherPrecpCountByKey.reduceByKey((v1,v2) => (v1._1+v2._1, v1._2+v2._2))\n    //   println(weatherPrecpAddByKey.first())\n      val weatherAverage = weatherPrecpAddByKey.map( k => (k._1, k._2._1 / (k._2._2).toFloat ) )\n    //   println(weatherAverages.first())\n      val weatherAverages = weatherAverage.map{case (x,y) => (y,x)}\n    //   val averages = weatherAverages\n     weatherAverages.top(10).mkString(\"\\n\")\n    \n      case class Weather(average: Float, station:String)\n      val map = weatherAverages.map(e =>Weather(e._1.toFloat,e._2))\n      val weather = map.toDF().registerTempTable(\"weather\")\n","user":"anonymous","dateUpdated":"2017-09-20T00:15:19-0700","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"ERROR","msg":[{"type":"TEXT","data":"\nimport org.apache.spark.SparkContext\n\nimport org.apache.spark.SparkContext._\n\nimport org.apache.spark.SparkConf\n\nimport java.net.URL\n\nimport java.nio.charset.Charset\n\nimport org.apache.commons.io.IOUtils\n\nimport org.apache.spark.sql.SparkSession\n\nimport spark.implicits._\n\nconf: org.apache.spark.SparkConf = org.apache.spark.SparkConf@6e6a552a\n\nres37: org.apache.spark.SparkConf = org.apache.spark.SparkConf@6e6a552a\n\nsc: org.apache.spark.SparkContext = org.apache.spark.SparkContext@b58978a\n\nweather: org.apache.spark.rdd.RDD[String] = /Users/kemy/Downloads/2015.csv MapPartitionsRDD[1] at textFile at <console>:73\n\nweatherParse: org.apache.spark.rdd.RDD[Array[String]] = MapPartitionsRDD[2] at map at <console>:76\n\nweatherPrecp: org.apache.spark.rdd.RDD[Array[String]] = MapPartitionsRDD[3] at filter at <console>:78\n\nweatherPrecpCountByKey: org.apache.spark.rdd.RDD[(String, (Int, Int))] = MapPartitionsRDD[4] at map at <console>:80\n\nweatherPrecpAddByKey: org.apache.spark.rdd.RDD[(String, (Int, Int))] = ShuffledRDD[5] at reduceByKey at <console>:82\n\nweatherAverage: org.apache.spark.rdd.RDD[(String, Float)] = MapPartitionsRDD[6] at map at <console>:84\n\nweatherAverages: org.apache.spark.rdd.RDD[(Float, String)] = MapPartitionsRDD[7] at map at <console>:86\n\n\n\n\n\n\n\n\n\n\n\nres45: String =\n(1524.0,US1MSWR0010)\n(1229.0,PPM00092100)\n(1219.0,US1TXGG0014)\n(1069.0,US1GACE0029)\n(1054.0,US1ARCW0010)\n(978.0,US1MSWB0004)\n(975.0,US1ARSH0010)\n(907.0,US1ALCH0003)\n(894.0,US1GABT0001)\n(815.0,US1WAWC0001)\n\ndefined class Weather\n\nmap: org.apache.spark.rdd.RDD[Weather] = MapPartitionsRDD[9] at map at <console>:89\n\nwarning: there was one deprecation warning; re-run with -deprecation for details\n\nweather: Unit = ()\n\nweatherAnalysis: org.apache.spark.sql.DataFrame = [average: float, station: string]\n\n\n\n<console>:72: error: value plot is not a member of org.apache.spark.sql.DataFrame\n             weatherAnalysis.plot()\n                             ^\n"}]},"apps":[],"jobName":"paragraph_1505891168605_-1513561627","id":"20170920-000608_1686304438","dateCreated":"2017-09-20T00:06:08-0700","dateStarted":"2017-09-20T00:14:51-0700","dateFinished":"2017-09-20T00:15:05-0700","status":"ERROR","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:2006"},{"user":"anonymous","config":{"colWidth":12,"enabled":true,"results":{"0":{"graph":{"mode":"pieChart","height":300,"optionOpen":true,"setting":{"multiBarChart":{}},"commonSetting":{},"keys":[{"name":"station","index":0,"aggr":"sum"}],"groups":[],"values":[{"name":"average","index":1,"aggr":"sum"}]},"helium":{}}},"editorSetting":{"language":"sql"},"editorMode":"ace/mode/sql"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1505891203787_-1663800877","id":"20170920-000643_1870888598","dateCreated":"2017-09-20T00:06:43-0700","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2007","text":"%sql \nselect station, average\nfrom weather \nwhere average > 800.0\ngroup by station, average\norder by average","dateUpdated":"2017-09-20T00:19:26-0700","dateFinished":"2017-09-20T00:18:40-0700","dateStarted":"2017-09-20T00:18:39-0700","results":{"code":"SUCCESS","msg":[{"type":"TABLE","data":"station\taverage\nUS1WAWC0001\t815.0\nUS1GABT0001\t894.0\nUS1ALCH0003\t907.0\nUS1ARSH0010\t975.0\nUS1MSWB0004\t978.0\nUS1ARCW0010\t1054.0\nUS1GACE0029\t1069.0\nUS1TXGG0014\t1219.0\nPPM00092100\t1229.0\nUS1MSWR0010\t1524.0\n"}]}},{"text":"%sql\n","user":"anonymous","dateUpdated":"2017-09-20T00:15:39-0700","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"sql"},"editorMode":"ace/mode/sql"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1505891739481_-1359680157","id":"20170920-001539_1430473667","dateCreated":"2017-09-20T00:15:39-0700","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:2147"}],"name":"WeatherDataAnalysis","id":"2CUMQG455","angularObjects":{"2CS5A2FF8:shared_process":[],"2CUUA9KVY:shared_process":[],"2CVGCPRB5:shared_process":[],"2CTFRWDWD:shared_process":[],"2CUMJ4GF4:shared_process":[],"2CT5QBRGR:shared_process":[],"2CVCR3FEV:shared_process":[],"2CSXSBT44:shared_process":[],"2CTEQSTVG:shared_process":[],"2CVD8YT32:shared_process":[],"2CS3ZKCUW:shared_process":[],"2CTCMWE32:shared_process":[],"2CVJ4T9H4:shared_process":[],"2CVE9GBR4:shared_process":[],"2CU3SQ155:shared_process":[],"2CTHC3ZMS:shared_process":[],"2CSPJ9ATX:shared_process":[],"2CU2UCW2G:shared_process":[],"2CUBA59RB:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}