{"id":1649,"date":"2015-12-22T11:15:57","date_gmt":"2015-12-22T03:15:57","guid":{"rendered":"http:\/\/blog2.tibame.com\/?p=1649"},"modified":"2022-06-08T09:56:05","modified_gmt":"2022-06-08T01:56:05","slug":"%e5%9c%a8%e7%86%9f%e6%82%89hadoop%e5%9f%ba%e7%a4%8e%e6%a6%82%e5%bf%b5%e4%bb%a5%e5%be%8c%ef%bc%8c%e5%a4%a7%e6%95%b8%e6%93%9a%e7%9a%84%e4%b8%8b%e4%b8%80%e6%ad%a5-apache-spark","status":"publish","type":"post","link":"https:\/\/blog.tibame.com\/?p=1649","title":{"rendered":"\u5728\u719f\u6089Hadoop\u57fa\u790e\u6982\u5ff5\u4ee5\u5f8c\uff0c\u5927\u6578\u64da\u7684\u4e0b\u4e00\u6b65&#8211;Apache Spark"},"content":{"rendered":"<p><strong>\u4ecb\u7d39:<br \/>\n<\/strong><a href=\"http:\/\/www.tibame.com\/q?pg=programs_all&amp;tg=ProgramAll-Welcome&amp;cx=22.20000&amp;programId=30000000258\">Spark\u5b78\u7a0b<\/a>\u5305\u542b\u4e86\u89e3Spark\u6838\u5fc3\u57fa\u790e\u8207\u74b0\u5883\u5efa\u7f6e\u3001\u5b78\u7fd2Scala\u8a9e\u8a00\u4ee5\u638c\u63e1Spark\u7cbe\u9ad3\u3002\u5f9e\u8cc7\u6599\u6536\u96c6\u958b\u59cb\uff0c\u5be6\u505aStreaming\u4e32\u6d41\u5206\u6790\uff0c\u5c07web log\u8cc7\u6599\u4ee5realtime (micro batch)\u65b9\u5f0f\u6536\u9032HDFS\u5132\u5b58\uff0c\u4ee5Spark SQL\u5c0dStreaming\u6536\u5165\u7684log\u8cc7\u6599\u505a\u7d50\u69cb\u5316\u8655\u7406\uff0c\u4ee5\u53ca\u4f7f\u7528Spark\u5167\u5efa\u6a5f\u5668\u5b78\u7fd2\u5957\u4ef6\u5b8c\u6210\u63a8\u85a6\u6a21\u578b\u3002\u9019\u5b78\u7a0b\u7121\u7591\u662f\u8b93\u4f60\u8f15\u9b06\u9032\u5165\u5927\u6578\u64da\u5206\u6790\u9818\u57df\u7684\u6572\u9580\u78da\u3002<br \/>\n<!--more--><br \/>\n<span style=\"text-align: center;\"><iframe src=\"https:\/\/www.youtube.com\/embed\/lhlHrr1-3mA\" width=\"800\" height=\"450\" frameborder=\"0\" allowfullscreen=\"allowfullscreen\"><\/iframe><\/span><br \/>\n<strong>OnO\u5b78\u7fd2\u4ecb\u7d39:<\/strong><br \/>\n<span style=\"color: #0066cc;\">\u76ee\u524d\u63a8\u52d5OnO\u5b78\u7fd2\u6a21\u5f0f,\u60a8\u9700\u5148\u4e0a\u5b8c\u7dda\u4e0a\u8ab2\u7a0b,\u518d\u4f86\u53c3\u52a0\u5be6\u4f5c\u8ab2\u7a0b,\u7528\u610f\u5728\u65bc<br \/>\n=&gt;\u5be6\u4f5c\u8ab2\u7a0b\u4e3b\u8981\u662f\u5728\u6559\u5c0e\u5b78\u54e1,\u5982\u4f55\u904b\u7528Spark\u6280\u8853\u4f86\u8655\u7406\u5927\u6578\u64da\u554f\u984c,\u8b93\u5b78\u54e1\u4e86\u89e3Spark\u7684\u8655\u7406\u6d41\u7a0b\u8207\u67b6\u69cb<br \/>\n=&gt;\u7dda\u4e0a\u8ab2\u7a0b\u5728\u8b1b\u89e3Spark\u7684\u76f8\u95dc\u77e5\u8b58,\u8209\u51e1\u5b89\u88dd,scala\u7a0b\u5f0f\u7b49\u7b49\u7684\u57fa\u790e\u77e5\u8b58<br \/>\n<\/span><\/p>\n<hr \/>\n<p><strong>\u7dda\u4e0a\u8ab2\u7a0b\u5167\u5bb9:<\/strong><\/p>\n<div style=\"background-color: #ffd306; text-align: center; font-size: 24px;\"><a href=\"http:\/\/www.tibame.com\/q?pg=programs_all&amp;tg=ProgramAll-Welcome&amp;cx=22.20000&amp;programId=30000000258\">Apache Spark \u5927\u6578\u64da\u61c9\u7528\u5de5\u7a0b\u5e2b<\/a><\/div>\n<div class=\"pcrstb-wrap\"><table style=\"border: 1px #C6A300 solid;\">\n<tbody>\n<tr>\n<td style=\"background-color: #fff4c1; font-size: 20px;\" width=\"25%\">\u8ab2\u7a0b\u540d\u7a31<\/td>\n<td style=\"background-color: #fff4c1; font-size: 20px;\">\u8ab2\u7a0b\u5167\u5bb9\u4ecb\u7d39<\/td>\n<\/tr>\n<tr>\n<td style=\"background-color: #fff4c1;\"><strong><a href=\"http:\/\/www.tibame.com\/q?pg=oocourses_all&amp;tg=OOCourseAll-Welcome&amp;cx=22.20000-232.30000004314&amp;programId=30000000258\">Spark \u5b89\u88dd\u7c21\u4ecb\u8207\u539f\u7406<br \/>\n(77 min)<\/a><\/strong><\/td>\n<td style=\"text-align: left; font-size: 14px;\"><span style=\"text-align: left;\">Spark\u57fa\u65bc\u8a18\u61b6\u9ad4\u904b\u7b97\uff0c\u76f8\u8f03\u65bcHadoop\u4e0a\u7684MapReduce\u6548\u80fd\u5feb\u4e0a\u8a31\u591a\uff0c\u8ab2\u7a0b\u4e2d\u6703\u4ecb\u7d39Spark\u6574\u9ad4\u67b6\u69cb\u8a2d\u8a08\u539f\u7406\u4ee5\u53ca\u5982\u4f55\u5efa\u7f6eSpark\u74b0\u5883\uff1b\u6700\u5f8c\uff0c\u7576\u7136\u4e0d\u6703\u932f\u904eSpark\u4e2d\u6700\u57fa\u790e\u4e5f\u662f\u6700\u91cd\u8981\u7684\u8cc7\u6599\u7d50\u69cb\uff0dRDD\uff0c\u6211\u5011\u5c07\u4e00\u8d77\u5b78\u7fd2\u4ec0\u9ebc\u662fRDD\uff0c\u4ee5\u53ca\u53c8\u8a72\u5982\u4f55\u6700\u6709\u6548\u7387\u5730\u64cd\u4f5cRDD\u3002<\/span><\/td>\n<\/tr>\n<tr>\n<td style=\"background-color: #fff4c1;\"><strong><a href=\"http:\/\/www.tibame.com\/q?pg=oocourses_all&amp;tg=OOCourseAll-Welcome&amp;cx=22.20000-232.30000004315&amp;programId=30000000258\">Spark \u6838\u5fc3\u8a9e\u8a00Scala<br \/>\n(66 min)<\/a><\/strong><\/td>\n<td style=\"background-color: white; text-align: left; font-size: 14px;\">Scala\u4f5c\u70baSpark\u7684\u539f\u751f\u8a9e\u8a00\uff0c\u4e86\u89e3Scala\u4e00\u4e9b\u57fa\u672c\u8a9e\u6cd5\uff0c\u6709\u52a9\u65bc\u7406\u89e3Spark\u7684\u64cd\u4f5c\u3002\u5728\u9019\u500b\u8ab2\u7a0b\u4e2d\u6211\u5011\u5c07\u4ecb\u7d39Scala\u4e00\u4e9b\u7279\u9ede\uff0c\u5305\u62ecOO\u53caFP\u7684\u5be6\u73fe\u4ee5\u53caScala\u7684\u7de8\u8b6f\u5668\uff0dSBT\u3002<\/td>\n<\/tr>\n<tr>\n<td style=\"background-color: #fff4c1;\"><strong><a href=\"http:\/\/www.tibame.com\/q?pg=oocourses_all&amp;tg=OOCourseAll-Welcome&amp;cx=22.20000-232.30000004316&amp;programId=30000000258\">\u7528Spark Streaming\u8b80\u53d6\u5373\u6642\u8cc7\u6599<br \/>\n(40 min)<\/a><\/strong><\/td>\n<td style=\"text-align: left; font-size: 14px;\">Spark Streaming\u662fSpark\u5ef6\u4f38\u7684API\uff0c\u5177\u5099\u53ef\u64f4\u5c55\uff0c\u9ad8\u541e\u5410\u53ca\u5bb9\u932f\u7684\u6d41\u5f0f\u8655\u7406\u67b6\u69cb\u3002\u5728\u9019\u5802\u8ab2\uff0c\u5c07\u6703\u5b78\u5230Streaming\u67b6\u69cb\uff0cDStream\u7684\u64cd\u4f5c\uff0c\u4ee5\u53ca\u8207Kafka\u7684\u6574\u5408\u3002<\/td>\n<\/tr>\n<tr>\n<td style=\"background-color: #fff4c1;\"><strong><a href=\"http:\/\/www.tibame.com\/q?pg=oocourses_all&amp;tg=OOCourseAll-Welcome&amp;cx=22.20000-232.30000004317&amp;programId=30000000258\">Spark SQL\u8655\u7406\u534a\u7d50\u69cb\u5316\u8cc7\u6599<br \/>\n(75 min)<\/a><\/strong><\/td>\n<td style=\"background-color: white; text-align: left; font-size: 14px;\">\u5229\u7528Spark SQL\u8207\u5404\u5f0f\u8cc7\u6599\u4e32\u9023\uff0c\u4e0d\u8ad6\u662fHDFS\u3001\u672c\u6a5f\u7cfb\u7d71\u6216\u662fHadoop Hive\u8cc7\u6599\u8868\uff0c\u900f\u904e\u53d6\u5f97\u8cc7\u6599\u3001\u8cc7\u6599\u524d\u8655\u7406\u3001\u5229\u7528SQL\u8a9e\u8a00\u8207DataFrame\u7684\u512a\u9ede\uff0c\u88fd\u4f5c\u5546\u696d\u667a\u6167\u5831\u8868\uff0c\u751a\u81f3\u4e86\u89e3Spark SQL\u6548\u80fd\u8abf\u6574\u7684\u529f\u80fd\u8a2d\u5b9a\uff0c\u8b93\u4f60\u8f15\u9b06\u638c\u63e1Spark SQL\u7684\u7cbe\u9ad3!<\/td>\n<\/tr>\n<tr>\n<td style=\"background-color: #fff4c1;\"><strong><a href=\"http:\/\/www.tibame.com\/q?pg=oocourses_all&amp;tg=OOCourseAll-Welcome&amp;cx=22.20000-232.30000004318&amp;programId=30000000258\">Spark MLlib\u5b8c\u6210\u8cc7\u6599\u6316\u6398\u8207\u6a5f\u5668\u5b78\u7fd2<br \/>\n(88 min)<\/a><\/strong><\/td>\n<td style=\"text-align: left; font-size: 14px;\">\u672c\u8ab2\u7a0b\u5167\u5bb9\u6db5\u84cb\u6a5f\u5668\u5b78\u7fd2\u3001\u8cc7\u6599\u8655\u7406\uff0c\u4ee5\u53ca\u8cc7\u6599\u5206\u6790\u3002\u5c07\u900f\u904e\u6df1\u5165\u6dfa\u51fa\u7684\u65b9\u5f0f\u4e86\u89e3\u6a5f\u5668\u5b78\u7fd2\u539f\u7406\uff0c\u4f7f\u7528Spark MLlib\u5206\u6790\u771f\u5be6\u8cc7\u6599\uff0c\u5b78\u7fd2\u4e4b\u5f8c\u80fd\u7acb\u5373\u61c9\u7528\u65bc\u65e5\u5e38\u5de5\u4f5c\u4e0a\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table><\/div>\n<hr \/>\n<p><strong>\u7dda\u4e0b\u5be6\u4f5c\u8ab2\u7a0b:<\/strong><br \/>\n<strong>\u4e3b\u984c:<\/strong>\u7db2\u8def\u65e5\u8a8c\u5be6\u6642\u6d41\u5206\u6790\u5be6\u6230\u6559\u5b78<br \/>\n<strong>OnO\u8ab2\u7a0b\u50f9\u683c:<\/strong>\u539f\u50f9NT$7395,<span style=\"color: red;\">\u512a\u60e0\u50f96200(\u671f\u9650\u81f312\/25)<\/span>-&gt;<a href=\"http:\/\/www.tibame.com\/q?pg=offlineClasses_all&amp;tg=OfflineClassesSummary-Welcome&amp;cx=22.20000-232.30000004742\">\u9ede\u6211\u5831\u540d<\/a><br \/>\n<strong>\u8ab2\u7a0b\u8b1b\u5e2b:<\/strong><\/p>\n<div class=\"pcrstb-wrap\"><table>\n<tbody>\n<tr>\n<td style=\"background-color: #ffd306; text-align: center; font-size: 20px;\" width=\"30%\">\u8b1b\u5e2b<\/td>\n<td style=\"background-color: #ffd306; text-align: center; font-size: 20px;\">\u4ecb\u7d39<\/td>\n<\/tr>\n<tr>\n<td style=\"font-size: 16px;\">\u694a\u7acb\u5049 Bryan Yang<\/td>\n<td style=\"font-size: 14px; text-align: left;\">\u73fe\u8077\u70ba\u5927\u6578\u64da\u8cc7\u6599\u5de5\u7a0b\u5e2b\uff0c\u66fe\u4efb\u8077\u96fb\u4fe1\u696d\u53ca\u5916\u5546\u9867\u554f\u516c\u53f8\u3002\u73fe\u6d3b\u8e8d\u65bcSpark\u53f0\u7063\u793e\u7fa4\uff0c\u81f4\u529b\u65bc\u63a8\u5ee3Spark\u6280\u8853\u8207\u5546\u696d\u61c9\u7528\u6574\u5408\u3002<\/td>\n<\/tr>\n<tr>\n<td style=\"background-color: white; font-size: 16px;\">\u674e\u4fd0\u6167 Li-Hui Li<\/td>\n<td style=\"background-color: white; font-size: 14px; text-align: left;\">\u66fe\u4efb\u91ab\u9662\u81e8\u5e8a\u8a66\u9a57\u7d71\u8a08\u5206\u6790\u5e2b\u3001\u96fb\u5b50\u5546\u52d9\u8cc7\u6599\u63a1\u7926\u5206\u6790\u5e2b\u3001\u884c\u92b7\u9867\u554f\u8cc7\u6599\u5de5\u7a0b\u5e2b\uff0c\u73fe\u6d3b\u8e8d\u65bcSpark\u53f0\u7063\u793e\u7fa4\uff0c\u81f4\u529b\u65bc\u63a8\u5ee3Spark\u6280\u8853\u8207\u5546\u696d\u61c9\u7528\u6574\u5408\u3002<\/td>\n<\/tr>\n<tr>\n<td style=\"font-size: 16px;\">\u694a\u64c7\u4e2d Mark Yang<\/td>\n<td style=\"font-size: 14px; text-align: left;\">\u7d71\u8a08\u672c\u79d1\u7562\u696d\uff0c\u7562\u696d\u5f8c\u8aa4\u5165\u7a0b\u5f0f\u53e2\u6797\uff0c\u71b1\u611b\u8edf\u9ad4\u67b6\u69cb\u8a2d\u8a08\u53ca\u6578\u64da\u5206\u6790\u3002\u73fe\u4efb\u624b\u6a5f\u8cc7\u6599\u5206\u6790\u5de5\u7a0b\u5e2b\uff0c\u70baHadoopCon2015\u7684\u8b1b\u8005\uff0c\u76ee\u524d\u5e38\u51fa\u6c92\u65bcSpark\u53f0\u7063\u793e\u7fa4\uff0c\u81f4\u529b\u65bc\u63a8\u5ee3Spark\u6280\u8853\u8207Scala\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table><\/div>\n<p>\u5099\u8a3b:\u7686\u6d3b\u8e8d\u65bcSpark\u53f0\u7063\u793e\u7fa4\uff0c\u81f4\u529b\u65bc\u63a8\u5ee3Spark\u6280\u8853\u8207\u5546\u696d\u61c9\u7528\u6574\u5408\u3002<br \/>\n<strong>\u6642\u9593:<\/strong>1\/10-&gt;<a href=\"http:\/\/www.tibame.com\/q?pg=offlineClasses_all&amp;tg=OfflineClassesSummary-Welcome&amp;cx=22.20000-232.30000004742\">\u9ede\u6211\u5831\u540d<\/a><br \/>\n<strong>\u5730\u9ede:<\/strong>\u53f0\u5317\u5546\u696d\u6280\u8853\u5927\u5b78<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u4ecb\u7d39: Spark\u5b78\u7a0b\u5305\u542b\u4e86\u89e3Spark\u6838\u5fc3\u57fa\u790e&hellip;<\/p>\n","protected":false},"author":19,"featured_media":1650,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_lmt_disableupdate":"","_lmt_disable":"","footnotes":""},"categories":[480,173,1543],"tags":[660,661],"class_list":["post-1649","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-technology","category-173","category-1543","tag-hadoop","tag-apache-spark"],"aioseo_notices":[],"views":5191,"amp_enabled":true,"_links":{"self":[{"href":"https:\/\/blog.tibame.com\/index.php?rest_route=\/wp\/v2\/posts\/1649","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.tibame.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.tibame.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.tibame.com\/index.php?rest_route=\/wp\/v2\/users\/19"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.tibame.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1649"}],"version-history":[{"count":3,"href":"https:\/\/blog.tibame.com\/index.php?rest_route=\/wp\/v2\/posts\/1649\/revisions"}],"predecessor-version":[{"id":1653,"href":"https:\/\/blog.tibame.com\/index.php?rest_route=\/wp\/v2\/posts\/1649\/revisions\/1653"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/blog.tibame.com\/index.php?rest_route=\/wp\/v2\/media\/1650"}],"wp:attachment":[{"href":"https:\/\/blog.tibame.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1649"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.tibame.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1649"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.tibame.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1649"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}