Repository navigation
Expand file tree
/
Copy pathdb.json
More file actions
1 lines (1 loc) · 167 KB
/
Copy pathdb.json
File metadata and controls
1 lines (1 loc) · 167 KB
1
{"meta":{"version":1,"warehouse":"5.0.1"},"models":{"Asset":[{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","path":"css/style.styl","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","path":"fancybox/jquery.fancybox.min.css","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","path":"fancybox/jquery.fancybox.min.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.6.4.min.js","path":"js/jquery-3.6.4.min.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","path":"js/script.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","path":"css/images/banner.jpg","modified":0,"renderable":1},{"_id":"themes/fluid/source/css/gitalk.css","path":"css/gitalk.css","modified":0,"renderable":1},{"_id":"themes/fluid/source/css/highlight-dark.styl","path":"css/highlight-dark.styl","modified":0,"renderable":1},{"_id":"themes/fluid/source/css/highlight.styl","path":"css/highlight.styl","modified":0,"renderable":1},{"_id":"themes/fluid/source/css/main.styl","path":"css/main.styl","modified":0,"renderable":1},{"_id":"themes/fluid/source/xml/local-search.xml","path":"xml/local-search.xml","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/boot.js","path":"js/boot.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/events.js","path":"js/events.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/color-schema.js","path":"js/color-schema.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/img-lazyload.js","path":"js/img-lazyload.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/plugins.js","path":"js/plugins.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/local-search.js","path":"js/local-search.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/umami-view.js","path":"js/umami-view.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/utils.js","path":"js/utils.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/avatar.png","path":"img/avatar.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/default.png","path":"img/default.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/js/leancloud.js","path":"js/leancloud.js","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/loading.gif","path":"img/loading.gif","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/police_beian.png","path":"img/police_beian.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/fluid.png","path":"img/fluid.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/vectorzyj.jpg","path":"img/vectorzyj.jpg","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/wechat.jpg","path":"img/wechat.jpg","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/yz_favicon.png","path":"img/yz_favicon.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/favicon_io/about.txt","path":"img/favicon_io/about.txt","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/favicon_io/android-chrome-512x512.png","path":"img/favicon_io/android-chrome-512x512.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/favicon_io/apple-touch-icon.png","path":"img/favicon_io/apple-touch-icon.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/favicon_io/favicon-16x16.png","path":"img/favicon_io/favicon-16x16.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/favicon_io/favicon-32x32.png","path":"img/favicon_io/favicon-32x32.png","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/favicon_io/favicon.ico","path":"img/favicon_io/favicon.ico","modified":0,"renderable":1},{"_id":"themes/fluid/source/img/favicon_io/site.webmanifest","path":"img/favicon_io/site.webmanifest","modified":0,"renderable":1},{"_id":"source/img/MapReduce/mapreduce_workflow.png","path":"img/MapReduce/mapreduce_workflow.png","modified":0,"renderable":0},{"_id":"source/img/MapReduce/mapreduce_example.png","path":"img/MapReduce/mapreduce_example.png","modified":0,"renderable":0},{"_id":"themes/fluid/source/img/background.jpg","path":"img/background.jpg","modified":0,"renderable":1},{"_id":"source/img/MapReduce/datalocality.png","path":"img/MapReduce/datalocality.png","modified":0,"renderable":0},{"_id":"source/img/MapReduce/yarnlayer.png","path":"img/MapReduce/yarnlayer.png","modified":0,"renderable":0},{"_id":"source/img/MapReduce/yarnapp.png","path":"img/MapReduce/yarnapp.png","modified":0,"renderable":0},{"_id":"source/img/MapReduce/mapreduce_jobexec.png","path":"img/MapReduce/mapreduce_jobexec.png","modified":0,"renderable":0},{"_id":"source/img/MapReduce/shufflesort.png","path":"img/MapReduce/shufflesort.png","modified":0,"renderable":0}],"Cache":[{"_id":"source/_posts/hello-world.md","hash":"7d98d6592de80fdcd2949bd7401cec12afd98cdf","modified":1727569792740},{"_id":"node_modules/hexo-theme-landscape/package.json","hash":"4bf95d52f77edf811f23f6d264a7493311a8d078","modified":1727569802277},{"_id":"node_modules/hexo-theme-landscape/_config.yml","hash":"b608c1f1322760dce9805285a602a95832730a2e","modified":1727569802382},{"_id":"node_modules/hexo-theme-landscape/README.md","hash":"1a9b279e6dd29fd19245f913f0c4a316ffaa62db","modified":1727569802282},{"_id":"node_modules/hexo-theme-landscape/languages/de-DE.yml","hash":"d29d1c4256b7ed9df42f511c2ff0a23ad5fd6c1f","modified":1727569802385},{"_id":"node_modules/hexo-theme-landscape/LICENSE","hash":"c480fce396b23997ee23cc535518ffaaf7f458f8","modified":1727569801941},{"_id":"node_modules/hexo-theme-landscape/languages/de.yml","hash":"3ebf0775abbee928c8d7bda943c191d166ded0d3","modified":1727569802386},{"_id":"node_modules/hexo-theme-landscape/languages/en-GB.yml","hash":"ea5e6aee4cb14510793ac4593a3bddffe23e530c","modified":1727569802396},{"_id":"node_modules/hexo-theme-landscape/languages/en-US.yml","hash":"ea5e6aee4cb14510793ac4593a3bddffe23e530c","modified":1727569802411},{"_id":"node_modules/hexo-theme-landscape/languages/default.yml","hash":"ea5e6aee4cb14510793ac4593a3bddffe23e530c","modified":1727569802393},{"_id":"node_modules/hexo-theme-landscape/languages/es-ES.yml","hash":"7008a8fc91f18d2a735864817b8ebda30c7a2c66","modified":1727569802414},{"_id":"node_modules/hexo-theme-landscape/languages/es.yml","hash":"76edb1171b86532ef12cfd15f5f2c1ac3949f061","modified":1727569802416},{"_id":"node_modules/hexo-theme-landscape/languages/en.yml","hash":"3083f319b352d21d80fc5e20113ddf27889c9d11","modified":1727569802414},{"_id":"node_modules/hexo-theme-landscape/languages/fr-FR.yml","hash":"8d09dbdab00a30a2870b56f7c0a7ca7deafa7b88","modified":1727569802420},{"_id":"node_modules/hexo-theme-landscape/languages/fr.yml","hash":"415e1c580ced8e4ce20b3b0aeedc3610341c76fb","modified":1727569802421},{"_id":"node_modules/hexo-theme-landscape/languages/hu-HU.yml","hash":"712d18664898fa21ba38d4973e90ef41a324ea25","modified":1727569802422},{"_id":"node_modules/hexo-theme-landscape/languages/hu.yml","hash":"284d557130bf54a74e7dcef9d42096130e4d9550","modified":1727569802422},{"_id":"node_modules/hexo-theme-landscape/languages/it-IT.yml","hash":"2cb6dc2fab9bd2dbe1c8bb869a9e8bf85a564fdd","modified":1727569802423},{"_id":"node_modules/hexo-theme-landscape/languages/it.yml","hash":"89b7d91306b2c1a0f3ac023b657bf974f798a1e8","modified":1727569802425},{"_id":"node_modules/hexo-theme-landscape/languages/ja-JP.yml","hash":"08481267e0c112e1f6855620f2837ec4c4a98bbd","modified":1727569802428},{"_id":"node_modules/hexo-theme-landscape/languages/ko-KR.yml","hash":"19209ad8f9d4057e8df808937f950eb265e1db69","modified":1727569802429},{"_id":"node_modules/hexo-theme-landscape/languages/ja.yml","hash":"a73e1b9c80fd6e930e2628b393bfe3fb716a21a9","modified":1727569802429},{"_id":"node_modules/hexo-theme-landscape/languages/ko.yml","hash":"881d6a0a101706e0452af81c580218e0bfddd9cf","modified":1727569802430},{"_id":"node_modules/hexo-theme-landscape/languages/mn-MN.yml","hash":"b9e5f3e7c0c2f779cf2cfded6db847b5941637ca","modified":1727569802431},{"_id":"node_modules/hexo-theme-landscape/languages/mn.yml","hash":"2e7523951072a9403ead3840ad823edd1084c116","modified":1727569802433},{"_id":"node_modules/hexo-theme-landscape/languages/nl-NL.yml","hash":"5ebbc30021f05d99938f96dfff280392df7f91f0","modified":1727569802436},{"_id":"node_modules/hexo-theme-landscape/languages/nl.yml","hash":"12ed59faba1fc4e8cdd1d42ab55ef518dde8039c","modified":1727569802438},{"_id":"node_modules/hexo-theme-landscape/languages/no.yml","hash":"965a171e70347215ec726952e63f5b47930931ef","modified":1727569802438},{"_id":"node_modules/hexo-theme-landscape/languages/pt-PT.yml","hash":"0f852b6b228e6ea59aa3540574bb89b233f2a098","modified":1727569802438},{"_id":"node_modules/hexo-theme-landscape/languages/pt.yml","hash":"57d07b75d434fbfc33b0ddb543021cb5f53318a8","modified":1727569802439},{"_id":"node_modules/hexo-theme-landscape/languages/ru-RU.yml","hash":"360d11a28bb768afb1dd15f63fa7fd3a8cc547ee","modified":1727569802439},{"_id":"node_modules/hexo-theme-landscape/languages/ru.yml","hash":"4fda301bbd8b39f2c714e2c934eccc4b27c0a2b0","modified":1727569802440},{"_id":"node_modules/hexo-theme-landscape/languages/th-TH.yml","hash":"ebfdba9bc4842c829473c1e6e4544344f182724d","modified":1727569802441},{"_id":"node_modules/hexo-theme-landscape/languages/tr.yml","hash":"a1cdbfa17682d7a971de8ab8588bf57c74224b5b","modified":1727569802442},{"_id":"node_modules/hexo-theme-landscape/languages/th.yml","hash":"84a55b00aa01f03982be294e43c33a20e6d32862","modified":1727569802442},{"_id":"node_modules/hexo-theme-landscape/languages/zh-CN.yml","hash":"1efd95774f401c80193eac6ee3f1794bfe93dc5a","modified":1727569802443},{"_id":"node_modules/hexo-theme-landscape/languages/zh-TW.yml","hash":"53ce3000c5f767759c7d2c4efcaa9049788599c3","modified":1727569802444},{"_id":"node_modules/hexo-theme-landscape/layout/category.ejs","hash":"765426a9c8236828dc34759e604cc2c52292835a","modified":1727569802091},{"_id":"node_modules/hexo-theme-landscape/layout/archive.ejs","hash":"2703b07cc8ac64ae46d1d263f4653013c7e1666b","modified":1727569802078},{"_id":"node_modules/hexo-theme-landscape/layout/index.ejs","hash":"aa1b4456907bdb43e629be3931547e2d29ac58c8","modified":1727569802100},{"_id":"node_modules/hexo-theme-landscape/layout/layout.ejs","hash":"0d1765036e4874500e68256fedb7470e96eeb6ee","modified":1727569802102},{"_id":"node_modules/hexo-theme-landscape/scripts/fancybox.js","hash":"c857d7a5e4a5d71c743a009c5932bf84229db428","modified":1727569802149},{"_id":"node_modules/hexo-theme-landscape/layout/page.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1727569802106},{"_id":"node_modules/hexo-theme-landscape/layout/post.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1727569802107},{"_id":"node_modules/hexo-theme-landscape/layout/tag.ejs","hash":"eaa7b4ccb2ca7befb90142e4e68995fb1ea68b2e","modified":1727569802114},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/after-footer.ejs","hash":"377d257d5d16e0158a4405c72401517b074fd7ff","modified":1727569802018},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive-post.ejs","hash":"c7a71425a946d05414c069ec91811b5c09a92c47","modified":1727569802033},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/footer.ejs","hash":"3656eb692254346671abc03cb3ba1459829e0dce","modified":1727569802094},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/article.ejs","hash":"56597e951203dd662a6d2c817c7c4f1c920d4a25","modified":1727569802080},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/gauges-analytics.ejs","hash":"21a1e2a3907d1a3dad1cd0ab855fe6735f233c74","modified":1727569802096},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive.ejs","hash":"7cb70a7a54f8c7ae49b10d1f37c0a9b74eab8826","modified":1727569802037},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/mobile-nav.ejs","hash":"e952a532dfc583930a666b9d4479c32d4a84b44e","modified":1727569802103},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/google-analytics.ejs","hash":"2ea7442ea1e1a8ab4e41e26c563f58413b59a3d0","modified":1727569802097},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/archive.ejs","hash":"beb4a86fcc82a9bdda9289b59db5a1988918bec3","modified":1727569802061},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/recent_posts.ejs","hash":"60c4b012dcc656438ff59997e60367e5a21ab746","modified":1727569802109},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/head.ejs","hash":"f05bced793b0314d4f2ef0c993b3a51d0b7d203a","modified":1727569802098},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/category.ejs","hash":"dd1e5af3c6af3f5d6c85dfd5ca1766faed6a0b05","modified":1727569802087},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/header.ejs","hash":"6a5033d189554c9a6d42e2ef7952ae5c9742648e","modified":1727569802099},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tagcloud.ejs","hash":"b4a2079101643f63993dcdb32925c9b071763b46","modified":1727569802115},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tag.ejs","hash":"2de380865df9ab5f577f7d3bcadf44261eb5faae","modified":1727569802114},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/sidebar.ejs","hash":"930da35cc2d447a92e5ee8f835735e6fd2232469","modified":1727569802111},{"_id":"node_modules/hexo-theme-landscape/source/css/_extend.styl","hash":"222fbe6d222531d61c1ef0f868c90f747b1c2ced","modified":1727569802298},{"_id":"node_modules/hexo-theme-landscape/source/css/_variables.styl","hash":"ca28281423ae57d76b6c1eb91cd845fd4e518bd6","modified":1727569802302},{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","hash":"e55a1d92954ed20f6887f92dc727bb995a010a43","modified":1727569802380},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1727569801979},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","hash":"49773efcb2221bbdf2d86f3f5c5ff2d841b528cc","modified":1727569802264},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/gallery.ejs","hash":"3d9d81a3c693ff2378ef06ddb6810254e509de5b","modified":1727569802095},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/date.ejs","hash":"f1458584b679545830b75bef2526e2f3eb931045","modified":1727569802094},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/category.ejs","hash":"c6bcd0e04271ffca81da25bcff5adf3d46f02fc0","modified":1727569802085},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/title.ejs","hash":"4d7e62574ddf46de9b41605fe3140d77b5ddb26d","modified":1727569802116},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/nav.ejs","hash":"16a904de7bceccbb36b4267565f2215704db2880","modified":1727569802104},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/tag.ejs","hash":"2fcb0bf9c8847a644167a27824c9bb19ac74dd14","modified":1727569802112},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/archive.styl","hash":"db15f5677dc68f1730e82190bab69c24611ca292","modified":1727569802311},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/article.styl","hash":"2d1f6f79ebf9cb55ebdb3865a2474437eb2b37c6","modified":1727569802314},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/comment.styl","hash":"79d280d8d203abb3bd933ca9b8e38c78ec684987","modified":1727569802320},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/footer.styl","hash":"e35a060b8512031048919709a8e7b1ec0e40bc1b","modified":1727569802321},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/highlight.styl","hash":"9cc3b2927d814f2f6e8e188f9d3657b94f4c6ef3","modified":1727569802347},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/header.styl","hash":"268d2989acb06e2ddd06cc36a6918c6cd865476b","modified":1727569802345},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/mobile.styl","hash":"a399cf9e1e1cec3e4269066e2948d7ae5854d745","modified":1727569802357},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-bottom.styl","hash":"8fd4f30d319542babfd31f087ddbac550f000a8a","modified":1727569802377},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-aside.styl","hash":"890349df5145abf46ce7712010c89237900b3713","modified":1727569802374},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar.styl","hash":"404ec059dc674a48b9ab89cd83f258dec4dcb24d","modified":1727569802378},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/grid.styl","hash":"0bf55ee5d09f193e249083602ac5fcdb1e571aed","modified":1727569802336},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/mixin.styl","hash":"44f32767d9fd3c1c08a60d91f181ee53c8f0dbb3","modified":1727569802352},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.6.4.min.js","hash":"eda46747c71d38a880bee44f9a439c3858bb8f99","modified":1727569802195},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1727569802240},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1727569802146},{"_id":"source/_posts/test.md","hash":"0ec6e669b3002fc21ffef4245f86ab49a5f1e547","modified":1727570278956},{"_id":"source/.DS_Store","hash":"acbb7a9e34bb2beb4202219cc72e48d962151247","modified":1727902014920},{"_id":"public/2024/09/28/test/index.html","hash":"5d66a4f07f7140120bd794c2152f5ac4c73eb30d","modified":1727727207472},{"_id":"public/2024/09/28/hello-world/index.html","hash":"a38173492a116179c3e6fb0e5f826b2821f2dc23","modified":1727727207472},{"_id":"public/archives/index.html","hash":"ba4ba69645099a75a183d6868a59b30ddb080012","modified":1727727207472},{"_id":"public/archives/2024/index.html","hash":"9328ab69675029e07a759d9aadf1af99c2f6efcb","modified":1727727207472},{"_id":"public/archives/2024/09/index.html","hash":"5fcaacacbe099fef45d8e891fb9ab80d7d12b37e","modified":1727727207472},{"_id":"public/index.html","hash":"5d0dbe6456a95dbc20963f03b5942b5419b87ef6","modified":1727727207472},{"_id":"public/css/style.css","hash":"4b65ba99401dce8b954f99a41762967e21eb40c7","modified":1727576386665},{"_id":"public/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1727576386665},{"_id":"public/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1727576386665},{"_id":"public/js/jquery-3.6.4.min.js","hash":"eda46747c71d38a880bee44f9a439c3858bb8f99","modified":1727576386665},{"_id":"public/js/script.js","hash":"49773efcb2221bbdf2d86f3f5c5ff2d841b528cc","modified":1727576386665},{"_id":"public/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1727576386665},{"_id":"source/about/index.md","hash":"1901ca1194e189eaf99ae6179abf080bf237b314","modified":1727577037927},{"_id":"themes/fluid/source/css/_pages/_tag/tag.styl","hash":"da39a3ee5e6b4b0d3255bfef95601890afd80709","modified":1721800613000},{"_id":"themes/fluid/.editorconfig","hash":"33218fbd623feb43edf5f99f15965392cecc44a6","modified":1721800613000},{"_id":"themes/fluid/.eslintrc","hash":"4bc2b19ce2b8c4d242f97d4ccf2d741e68ab0097","modified":1721800613000},{"_id":"themes/fluid/.gitattributes","hash":"a54f902957d49356376b59287b894b1a3d7a003f","modified":1721800613000},{"_id":"themes/fluid/.gitignore","hash":"ae3bfcb89777657c5dfb5169d91445dcb0e5ab98","modified":1721800613000},{"_id":"themes/fluid/LICENSE","hash":"26f9356fd6e84b5a88df6d9014378f41b65ba209","modified":1721800613000},{"_id":"themes/fluid/package.json","hash":"7746460fc2eba7439b494c46aa9b5ded81370819","modified":1721800613000},{"_id":"themes/fluid/README.md","hash":"ff9b0e1fb9dba665af2f1e4a577f8cb9e840464b","modified":1721800613000},{"_id":"themes/fluid/README_en.md","hash":"365184a73af40e7365504c3077f3d80dfee1d80e","modified":1721800613000},{"_id":"themes/fluid/languages/de.yml","hash":"58dccef1d98b472dc4e6f4693c2297b0c9c5afba","modified":1721800613000},{"_id":"themes/fluid/_config.yml","hash":"592c1c3f334f99f843bce246ac4609aaa6ddbaea","modified":1727733761285},{"_id":"themes/fluid/languages/en.yml","hash":"9c580471257f5a32bee701a059a45ea96755dcdc","modified":1721800613000},{"_id":"themes/fluid/languages/eo.yml","hash":"7c1a0c9f6186b6643b19d3980f055329bdb4efa4","modified":1721800613000},{"_id":"themes/fluid/languages/ja.yml","hash":"550b95d3614a64592f02666938d235e9f11e449e","modified":1721800613000},{"_id":"themes/fluid/languages/es.yml","hash":"026ddf1a49bf8ddfef6ed86ab4d6af143c1dd95f","modified":1721800613000},{"_id":"themes/fluid/languages/zh-CN.yml","hash":"a60847136709bb95586a98d9d67b50390a8d2c96","modified":1721800613000},{"_id":"themes/fluid/languages/zh-HK.yml","hash":"51c2b4d64c6992a39bfd2586a1bdf5fbbbdf0175","modified":1721800613000},{"_id":"themes/fluid/languages/ru.yml","hash":"93818f8bf07195fb1ebffbb5210e531b0e3a6ec4","modified":1721800613000},{"_id":"themes/fluid/languages/zh-TW.yml","hash":"e1043de394f6dcf5c0647adcfdefe60637f78426","modified":1721800613000},{"_id":"themes/fluid/layout/404.ejs","hash":"b84d575c7b7f778b4cb64e89ad3d0aed4a896820","modified":1721800613000},{"_id":"themes/fluid/layout/about.ejs","hash":"052e9fc19c753f53fdc083c7fb098e3668880140","modified":1721800613000},{"_id":"themes/fluid/layout/archive.ejs","hash":"7c1f44005849791feae4abaa10fae4cb983d3277","modified":1721800613000},{"_id":"themes/fluid/layout/categories.ejs","hash":"13859726c27b6c79b5876ec174176d0f9c1ee164","modified":1721800613000},{"_id":"themes/fluid/layout/category.ejs","hash":"f099161b738a16a32253f42085b5444f902018ed","modified":1721800613000},{"_id":"themes/fluid/layout/index.ejs","hash":"33c3317cdcee062789de2336dd8d0cc7f86d3650","modified":1721800613000},{"_id":"themes/fluid/layout/layout.ejs","hash":"7e0023474128fbe4d68c467704c41f1712432415","modified":1721800613000},{"_id":"themes/fluid/layout/links.ejs","hash":"1cac32ec4579aaf7b9fa39d317497331d4c5e1dd","modified":1721800613000},{"_id":"themes/fluid/layout/page.ejs","hash":"ed5007a3feb8f14d3d2843271bfb298eb0c56219","modified":1721800613000},{"_id":"themes/fluid/layout/post.ejs","hash":"9bf0d357a607a282f3b9cb04525a4df0cc2a8b76","modified":1721800613000},{"_id":"themes/fluid/layout/tag.ejs","hash":"9d686364c4d16a1a9219471623af452035c5b966","modified":1721800613000},{"_id":"themes/fluid/layout/tags.ejs","hash":"1d06af34b6cf1d8a20d2eb565e309326ceba309f","modified":1721800613000},{"_id":"themes/fluid/.github/ISSUE_TEMPLATE/bug_report.md","hash":"554c0d0e086a0784d83ee71c83f8bceeb60aecc8","modified":1721800613000},{"_id":"themes/fluid/.github/ISSUE_TEMPLATE/bug_report_zh.md","hash":"c8b0d49c49e3c88872fd3b37909345ff5b2b6aa0","modified":1721800613000},{"_id":"themes/fluid/.github/ISSUE_TEMPLATE/feature_request.md","hash":"c134dd57ffd269b93402ccfffe7dbe0f0b583bec","modified":1721800613000},{"_id":"themes/fluid/.github/ISSUE_TEMPLATE/feature_request_zh.md","hash":"ed08574b196447376dd74411cca664ac9227a5d4","modified":1721800613000},{"_id":"themes/fluid/.github/ISSUE_TEMPLATE/question_zh.md","hash":"fff07ce0472afc368d388637cb9d438195da9b5b","modified":1721800613000},{"_id":"themes/fluid/scripts/events/index.js","hash":"79de5a379b28cad759a49048351c7f6b8915bd7d","modified":1721800613000},{"_id":"themes/fluid/.github/ISSUE_TEMPLATE/question.md","hash":"ab5eab9e3ff889c4ba7fd82846e7f5b7ae15bebc","modified":1721800613000},{"_id":"themes/fluid/scripts/filters/default-injects.js","hash":"b2013ae8e189cd07ebc8a2ff48a78e153345210f","modified":1721800613000},{"_id":"themes/fluid/scripts/filters/locals.js","hash":"58d0fec976f6b1d35e7ea03edc45414088acf05c","modified":1721800613000},{"_id":"themes/fluid/scripts/filters/post-filter.js","hash":"82bb06686158ebe160a631c79f156cd4fde35656","modified":1721800613000},{"_id":"themes/fluid/scripts/generators/index-generator.js","hash":"9159fc22fa84a7b605dd15fe4104f01fe9c71147","modified":1721800613000},{"_id":"themes/fluid/scripts/generators/local-search.js","hash":"9ac5ddad06e9b0e6015ce531430018182a4bc0fa","modified":1721800613000},{"_id":"themes/fluid/scripts/generators/pages.js","hash":"d3e75f53c59674d171309e50702954671f31f1a4","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/date.js","hash":"9bda6382f61b40a20c24af466fe10c8366ebb74c","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/engine.js","hash":"d3a231d106795ce99cb0bc77eb65f9ae44515933","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/export-config.js","hash":"8e67b522c47aa250860e3fe2c733f1f958a506c0","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/import.js","hash":"ca53e8dbf7d44cfd372cfa79ac60f35a7d5b0076","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/injects.js","hash":"1ad2ae6b11bd8806ee7dd6eb7140d8b54a95d613","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/page.js","hash":"4607607445233b3029ef20ed5e91de0da0a7f9c5","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/scope.js","hash":"d41d9d658fcb54964b388598e996747aadb85b0f","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/url.js","hash":"2a6a8288176d0e0f6ec008056bf2745a86e8943e","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/utils.js","hash":"966689d7c5e4320008285395fbaa2751f6209be5","modified":1721800613000},{"_id":"themes/fluid/scripts/helpers/wordcount.js","hash":"4d48c424e47ff9a17a563167ea5f480890267adf","modified":1721800613000},{"_id":"themes/fluid/.github/workflows/cr.yaml","hash":"19a8a00f5ba9607d82265572fe1202b64a8b0822","modified":1721800613000},{"_id":"themes/fluid/scripts/tags/button.js","hash":"3eb43a8cdea0a64576ad6b31b4df6c2bf5698d4c","modified":1721800613000},{"_id":"themes/fluid/.github/workflows/limit.yaml","hash":"f8bd2edeb4424ee7a055b31583445d5d5dff91a4","modified":1721800613000},{"_id":"themes/fluid/scripts/tags/checkbox.js","hash":"6eaf53cf4bfc756a65bda18184cf8998a12c861d","modified":1721800613000},{"_id":"themes/fluid/.github/workflows/publish.yaml","hash":"6f02e6440d88629229556e3fd47d0280fe2240db","modified":1721800613000},{"_id":"themes/fluid/scripts/tags/fold.js","hash":"73e4fd12ce3e47981479391ed354b7d9d3279f70","modified":1721800613000},{"_id":"themes/fluid/scripts/tags/group-image.js","hash":"4aeebb797026f1df25646a5d69f7fde79b1bcd26","modified":1721800613000},{"_id":"themes/fluid/scripts/tags/label.js","hash":"f05a6d32cca79535b22907dc03edb9d3fa2d8176","modified":1721800613000},{"_id":"themes/fluid/scripts/tags/mermaid.js","hash":"75160561e1ef3603b6d2ad2938464ab1cb77fd38","modified":1721800613000},{"_id":"themes/fluid/scripts/tags/note.js","hash":"e3b456a079e5dc0032473b516c865b20f83d2c26","modified":1721800613000},{"_id":"themes/fluid/scripts/utils/compare-versions.js","hash":"dbbc928c914fc2bd242cd66aa0c45971aec13a5d","modified":1721800613000},{"_id":"themes/fluid/scripts/utils/crypto.js","hash":"ae4ad8a188ef5b3fa6818b01629fc962b3de8551","modified":1721800613000},{"_id":"themes/fluid/scripts/utils/object.js","hash":"33b57e4decdc5e75c518859f168c8ba80b2c665b","modified":1721800613000},{"_id":"themes/fluid/scripts/utils/resolve.js","hash":"8c4a8b62aa8608f12f1e9046231dff04859dc3e9","modified":1721800613000},{"_id":"themes/fluid/scripts/utils/url-join.js","hash":"718aab5e7b2059a06b093ca738de420d9afa44ba","modified":1721800613000},{"_id":"themes/fluid/source/css/gitalk.css","hash":"a57b3cc8e04a0a4a27aefa07facf5b5e7bca0e76","modified":1721800613000},{"_id":"themes/fluid/source/css/highlight-dark.styl","hash":"45695ef75c31a4aa57324dd408b7e2327a337018","modified":1721800613000},{"_id":"themes/fluid/source/css/highlight.styl","hash":"a9efc52a646a9e585439c768557e3e3c9e3326dc","modified":1721800613000},{"_id":"themes/fluid/source/css/main.styl","hash":"855ae5fe229c51afa57f7645f6997a27a705d7e4","modified":1721800613000},{"_id":"themes/fluid/source/xml/local-search.xml","hash":"8c96ba6a064705602ce28d096fd7dd9069630a55","modified":1721800613000},{"_id":"themes/fluid/source/js/boot.js","hash":"38bd26c6b7acdafda86dda3560e6a3ca488d3c76","modified":1721800613000},{"_id":"themes/fluid/source/js/events.js","hash":"6869811f67e4c3de3edfa4b08464bb242b97a402","modified":1721800613000},{"_id":"themes/fluid/source/js/color-schema.js","hash":"1ef88c881b9f942deadde3d890387b94c617342a","modified":1721800613000},{"_id":"themes/fluid/source/js/img-lazyload.js","hash":"cbdeca434ec4da51f488c821d51b4d23c73294af","modified":1721800613000},{"_id":"themes/fluid/source/js/plugins.js","hash":"c34916291e392a774ff3e85c55badb83e8661297","modified":1721800613000},{"_id":"themes/fluid/source/js/local-search.js","hash":"b9945f76f8682f3ec32edfb285b26eb559f7b7e8","modified":1721800613000},{"_id":"themes/fluid/source/js/umami-view.js","hash":"33c4b3883fa747604074ad3921606eeeaeb50716","modified":1721800613000},{"_id":"themes/fluid/source/js/utils.js","hash":"b82e7c289a66dfd36064470fd41c0e96fc598b43","modified":1721800613000},{"_id":"themes/fluid/source/img/avatar.png","hash":"fe739a158cc128f70f780eb5fa96f388b81d478f","modified":1721800613000},{"_id":"themes/fluid/source/js/leancloud.js","hash":"eff77c7a5c399fcaefda48884980571e15243fc9","modified":1721800613000},{"_id":"themes/fluid/source/img/police_beian.png","hash":"90efded6baa2dde599a9d6b1387973e8e64923ea","modified":1721800613000},{"_id":"themes/fluid/source/img/loading.gif","hash":"2d2fc0f947940f98c21afafef39ecf226a2e8d55","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/archive-list.ejs","hash":"7520fbf91f762207c2ab06b2c293235cd5b23905","modified":1721800613000},{"_id":"themes/fluid/source/img/fluid.png","hash":"64b215db2cb3af98fe639e94537cb5209f959c78","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/category-chains.ejs","hash":"18309584aab83bc4deb20723ebad832149dd2e24","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/category-list.ejs","hash":"f8d2f1907450e61968e6d54443e9be8138196a77","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/css.ejs","hash":"1dadb118d580280524ed0a5f69bd34d234a92276","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/footer.ejs","hash":"40c8b0852873032e7aaef3f68e8ea08706cdef13","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments.ejs","hash":"d707c47b2638c94e489bc43d4cfd098b7c58447f","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/head.ejs","hash":"67be642f99482c07904474f410cfbc2f99003288","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/header.ejs","hash":"0d5e397d30051e5fbabe7b47cfd1f1e6a5820af1","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/markdown-plugins.ejs","hash":"fc4bdf7de0cf1a66d0e5e4fba1b31d6f7ed49468","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/paginator.ejs","hash":"0f38a2c238169edcb63fc46c23bfc529ff3859b7","modified":1721800613000},{"_id":"themes/fluid/scripts/events/lib/compatible-configs.js","hash":"ef474d1fa5bbafc52619ced0f9dc7eaf2affb363","modified":1721800613000},{"_id":"themes/fluid/scripts/events/lib/footnote.js","hash":"c19ac8050b82c3676b0332a56099ccfcc36d9d52","modified":1721800613000},{"_id":"themes/fluid/scripts/events/lib/hello.js","hash":"bd8376e1cf7892dc2daa58f2f443574be559fdbf","modified":1721800613000},{"_id":"themes/fluid/scripts/events/lib/injects.js","hash":"5ae4b07204683e54b5a1b74e931702bbce2ac23e","modified":1721800613000},{"_id":"themes/fluid/scripts/events/lib/highlight.js","hash":"a5fe1deccb73b5f578797dbb11038efc15f63ce8","modified":1721800613000},{"_id":"themes/fluid/scripts/events/lib/merge-configs.js","hash":"7c944c43b2ece5dd84859bd9d1fe955d13427387","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/scripts.ejs","hash":"da5810785105e5075861593c7ac22c7aa9665a72","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/search.ejs","hash":"70e1c929e084ca8a2648cedabf29b372511ea2b8","modified":1721800613000},{"_id":"themes/fluid/scripts/events/lib/lazyload.js","hash":"9ba0d4bc224e22af8a5a48d6ff13e5a0fcfee2a4","modified":1721800613000},{"_id":"themes/fluid/source/css/_variables/base.styl","hash":"4ed5f0ae105ef4c7dd92eaf652ceda176c38e502","modified":1721800613000},{"_id":"themes/fluid/source/css/_functions/base.styl","hash":"2e46f3f4e2c9fe34c1ff1c598738fc7349ae8188","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/pages.styl","hash":"b8e887bc7fb3b765a1f8ec9448eff8603a41984f","modified":1721800613000},{"_id":"themes/fluid/source/css/_mixins/base.styl","hash":"542e306ee9494e8a78e44d6d7d409605d94caeb3","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/changyan.ejs","hash":"c9b2d68ed3d375f1953e7007307d2a3f75ed6249","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/cusdis.ejs","hash":"5f9dc012be27040bbe874d0c093c0d53958cc987","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/disqus.ejs","hash":"aab4a4d24c55231a37db308ae94414319cecdd9b","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/discuss.ejs","hash":"98d065b58ce06b7d18bff3c974e96fa0f34ae03a","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/giscus.ejs","hash":"95f8b866b158eff9352c381c243b332a155a5110","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/livere.ejs","hash":"2264758fed57542a7389c7aa9f00f1aefa17eb87","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/gitalk.ejs","hash":"843bc141a4545eb20d1c92fb63c85d459b4271ec","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/remark42.ejs","hash":"d4e9532feeb02aed61bd15eda536b5b631454dac","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/utterances.ejs","hash":"c7ccf7f28308334a6da6f5425b141a24b5eca0e2","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/valine.ejs","hash":"19ba937553dddd317f827d682661a1066a7b1f30","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/waline.ejs","hash":"3d08c73b77e412d2f06a24d9344565fc7dbc76f8","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/footer/beian.ejs","hash":"4fb9b5dd3f3e41a586d6af44e5069afe7c81fff2","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/comments/twikoo.ejs","hash":"d84bcb5ccd78470a60c067fc914ac0ac67ac8777","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/footer/statistics.ejs","hash":"954a29b58d72647d20450da270b5d8fb2e0824f5","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/header/banner.ejs","hash":"e07757b59e7b89eea213d0e595cb5932f812fd32","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/header/navigation.ejs","hash":"37d750428772d7c71ba36ce0c2540780d90fadea","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/analytics.ejs","hash":"e6dcbf1c2f56314d56bb46b50aca86ff68cacebd","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/anchorjs.ejs","hash":"40181442d3a2b8734783a0ad7caf2d2522e3f2ab","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/code-widget.ejs","hash":"3a505cba37942badf62a56bbb8b605b72af330aa","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/encrypt.ejs","hash":"0fff24cf5bf99fbe5c56c292e2eac4a89bf29db4","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/fancybox.ejs","hash":"9d1ea2a46b8c8ad8c168594d578f40764818ef13","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/highlight.ejs","hash":"7529dd215b09d3557804333942377b9e20fa554e","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/math.ejs","hash":"dcbf9a381ee76f2f1f75fcbc22c50a502ec85023","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/mermaid.ejs","hash":"03ac02762f801970d1c4e73d6ec8d4c503780e50","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/moment.ejs","hash":"4ff3fb1b60ccc95a0af3bbdbd0757fedefc088b5","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/nprogress.ejs","hash":"4c2d39ce816b8a6dcd6b53113c8695f8bd650a23","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/plugins/typed.ejs","hash":"f345374885cd6a334f09a11f59c443b5d577c06c","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/post/category-bar.ejs","hash":"8772bce97ed297e7a88523f4e939ed6436c22f87","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/post/copyright.ejs","hash":"cbfa32c5f5973133afd043853b24f8200455cb2d","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/post/meta-bottom.ejs","hash":"375974ec017696e294dc12469fb0ae257800dc2d","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/post/meta-top.ejs","hash":"54dd479dbb440126e4ddd9d902229db5afaaae98","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/post/sidebar-left.ejs","hash":"9992c99b3eb728ad195970e1b84d665f2c8691c4","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/post/sidebar-right.ejs","hash":"d5fcc9b60e02f869a29a8c17a16a6028ecc1e6d8","modified":1721800613000},{"_id":"themes/fluid/layout/_partials/post/toc.ejs","hash":"635a89060fbf72eeda066fc4bd0a97462f069417","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_about/about.styl","hash":"97fe42516ea531fdad771489b68aa8b2a7f6ae46","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_archive/archive.styl","hash":"c475e6681546d30350eaed11f23081ecae80c375","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_index/index.styl","hash":"25fb6fa4c783b847c632584c49a7e1593cdb2f5d","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_category/category-bar.styl","hash":"cc6df43fef6bb3efecbfdd8b9e467424a1dea581","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_category/category-chain.styl","hash":"0cdf7ef50dfd0669d3b257821384ff31cd81b7c9","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_post/comment.styl","hash":"780f3788e7357bcd3f3262d781cb91bb53976a93","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_post/markdown.styl","hash":"1e3d3a82721e7c10bcfcecec6d81cf2979039452","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_links/links.styl","hash":"5c7f2044e3f1da05a3229537c06bd879836f8d6e","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_category/category-list.styl","hash":"7edfe1b571ecca7d08f5f4dbcf76f4ffdcfbf0b5","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_post/highlight.styl","hash":"4df764d298fe556e501db4afc2b05686fe6ebcfb","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_post/post-page.styl","hash":"7eee3f78296a3c81849a5415d1d43dcc6e03e6aa","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_post/post-tag.styl","hash":"c96d36aa8fe20f0c3c1a29ee2473cd8064b10f73","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/base.styl","hash":"643284c567665f96915f0b64e59934dda315f74d","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/color-schema.styl","hash":"85492ef64d7e5f70f0f7e46d570bbc911e686d7e","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/keyframes.styl","hash":"94065ea50f5bef7566d184f2422f6ac20866ba22","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/inline.styl","hash":"411a3fa3f924a87e00ff04d18b5c83283b049a4d","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_tag/tags.styl","hash":"65bfc01c76abc927fa1a23bf2422892b0d566c3f","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/print.styl","hash":"166afbc596ea4b552bad7290ec372d25ec34db7b","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/anchorjs.styl","hash":"e0cebda4a6f499aff75e71417d88caa7ceb13b94","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/board.styl","hash":"4397037fc3f0033dbe546c33cd9dbdabd8cb1632","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/banner.styl","hash":"7a0bd629bc234fc75e3cc8e3715ffada92f09e73","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/code-widget.styl","hash":"b66ab013f0f37d724a149b85b3c7432afcf460ad","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/copyright.styl","hash":"26f71a9cd60d96bb0cb5bbdf58150b8e524d9707","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/footer.styl","hash":"2caaca71dd1ff63d583099ed817677dd267b457e","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/header.styl","hash":"d42b748f2f49ef32aafb1a21d75991d2459da927","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/footnote.styl","hash":"ae9289cc89649af2042907f8a003303b987f3404","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/modal.styl","hash":"adf6c1e5c8e1fb41c77ce6e2258001df61245aa2","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/ngrogress.styl","hash":"5d225357b4a58d46118e6616377168336ed44cb2","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/noscript.styl","hash":"0cf2f2bb44f456150d428016675d5876a9d2e2aa","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/qrcode.styl","hash":"78704a94c0436097abfb0e0a57abeb3429c749b7","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/scroll-btn.styl","hash":"f0e429a27fa8a7658fcbddbb4d4dbe4afa12499a","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/search.styl","hash":"10f7e91a91e681fb9fe46f9df7707b9ef78707c8","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/toc.styl","hash":"9e7452aa2372153f25d7a4675c9d36d281a65d24","modified":1721800613000},{"_id":"themes/fluid/source/css/_pages/_base/_widget/pagination.styl","hash":"8bb1b68e5f3552cb48c2ffa31edbc53646a8fb4c","modified":1721800613000},{"_id":"themes/fluid/source/img/default.png","hash":"167a12978d80371cf578c8a2e45c24a2eb25b6fb","modified":1721800613000},{"_id":"public/local-search.xml","hash":"466cc5a49021c5cd4a4624f18ebab442a9b42d43","modified":1727727207472},{"_id":"public/about/index.html","hash":"2c06ae1defea39a54657ea74e0e3040b68d68a92","modified":1727727207472},{"_id":"public/404.html","hash":"0a5d22d0a2388a255fe0065987f44f7fcae2c5bf","modified":1727727207472},{"_id":"public/tags/index.html","hash":"be49f8cb1388f4702c40b67459a2afc30242a9de","modified":1727727207472},{"_id":"public/categories/index.html","hash":"60ea56a9dabab82887a2b20c73850547b7659c58","modified":1727727207472},{"_id":"public/links/index.html","hash":"cd653558a565fed990c5ecd8cb1e58627a355b90","modified":1727727207472},{"_id":"public/xml/local-search.xml","hash":"8c96ba6a064705602ce28d096fd7dd9069630a55","modified":1727576386665},{"_id":"public/img/avatar.png","hash":"fe739a158cc128f70f780eb5fa96f388b81d478f","modified":1727576386665},{"_id":"public/img/police_beian.png","hash":"90efded6baa2dde599a9d6b1387973e8e64923ea","modified":1727576386665},{"_id":"public/img/fluid.png","hash":"64b215db2cb3af98fe639e94537cb5209f959c78","modified":1727576386665},{"_id":"public/img/loading.gif","hash":"2d2fc0f947940f98c21afafef39ecf226a2e8d55","modified":1727576386665},{"_id":"public/css/gitalk.css","hash":"a57b3cc8e04a0a4a27aefa07facf5b5e7bca0e76","modified":1727576386665},{"_id":"public/css/highlight-dark.css","hash":"902294bada4323c0f51502d67cba8c3a0298952f","modified":1727576386665},{"_id":"public/css/main.css","hash":"14ebd9b515085666cee29bbcbe362ad3604ab62a","modified":1727576386665},{"_id":"public/css/highlight.css","hash":"04d4ddbb5e1d1007447c2fe293ee05aae9b9563e","modified":1727576386665},{"_id":"public/js/boot.js","hash":"38bd26c6b7acdafda86dda3560e6a3ca488d3c76","modified":1727576386665},{"_id":"public/js/color-schema.js","hash":"1ef88c881b9f942deadde3d890387b94c617342a","modified":1727576386665},{"_id":"public/js/img-lazyload.js","hash":"cbdeca434ec4da51f488c821d51b4d23c73294af","modified":1727576386665},{"_id":"public/js/events.js","hash":"6869811f67e4c3de3edfa4b08464bb242b97a402","modified":1727576386665},{"_id":"public/js/local-search.js","hash":"b9945f76f8682f3ec32edfb285b26eb559f7b7e8","modified":1727576386665},{"_id":"public/js/plugins.js","hash":"c34916291e392a774ff3e85c55badb83e8661297","modified":1727576386665},{"_id":"public/js/umami-view.js","hash":"33c4b3883fa747604074ad3921606eeeaeb50716","modified":1727576386665},{"_id":"public/js/utils.js","hash":"b82e7c289a66dfd36064470fd41c0e96fc598b43","modified":1727576386665},{"_id":"public/js/leancloud.js","hash":"eff77c7a5c399fcaefda48884980571e15243fc9","modified":1727576386665},{"_id":"public/img/default.png","hash":"167a12978d80371cf578c8a2e45c24a2eb25b6fb","modified":1727576386665},{"_id":"themes/fluid/.DS_Store","hash":"693632d7f815e8c48bfaaed4ffb2a5433d3b4179","modified":1727733554991},{"_id":"themes/fluid/source/.DS_Store","hash":"75337672632cdec64aa4f3045982ef030e72c684","modified":1727733554990},{"_id":"themes/fluid/source/img/vectorzyj.jpg","hash":"d9c78a9bd670f80a5f02e85bffc2502c7869fd93","modified":1727572784000},{"_id":"themes/fluid/source/img/wechat.jpg","hash":"ce74e7a179f4f84d2d184daaafcb9f154b4aa9eb","modified":1727572934000},{"_id":"public/img/vectorzyj.jpg","hash":"d9c78a9bd670f80a5f02e85bffc2502c7869fd93","modified":1727576386665},{"_id":"public/img/wechat.jpg","hash":"ce74e7a179f4f84d2d184daaafcb9f154b4aa9eb","modified":1727576386665},{"_id":"source/_posts/MapReduce.md","hash":"fe384a9e5d51cca7d9beba9aa5194a43e5cb4d41","modified":1727987892827},{"_id":"themes/fluid/source/img/yz_favicon.png","hash":"76b5f9b679da101aa92a32176115a52c1cda5e32","modified":1727593928000},{"_id":"themes/fluid/source/img/favicon_io/about.txt","hash":"d30c209db7ea73e07c38a05058382fcb33cf6df0","modified":1727593928000},{"_id":"themes/fluid/source/img/favicon_io/android-chrome-512x512.png","hash":"9602d6ace9a9b038addbf53e2c7a876b9a4a2abb","modified":1727593928000},{"_id":"themes/fluid/source/img/favicon_io/apple-touch-icon.png","hash":"220337f927628b442b446690dc13b8d454a9aa7a","modified":1727593928000},{"_id":"themes/fluid/source/img/favicon_io/favicon-16x16.png","hash":"3dd88a7c812be5cd897f810e32bdf55f7d872867","modified":1727593928000},{"_id":"themes/fluid/source/img/.DS_Store","hash":"f11cc275f638aa59c1448f2c9c9d9506fc29a9a3","modified":1727579638162},{"_id":"themes/fluid/source/img/favicon_io/favicon-32x32.png","hash":"ca31fa2f8dd242981e5a4058ab069be038d1b275","modified":1727593928000},{"_id":"themes/fluid/source/img/favicon_io/favicon.ico","hash":"3a7ed482c1d39e241e3b6923cf9c249e44d0b151","modified":1727593928000},{"_id":"themes/fluid/source/img/favicon_io/site.webmanifest","hash":"bf31baf91bdd2fcde24a45e3f2a1be33733c6f69","modified":1727593928000},{"_id":"themes/fluid/source/img/favicon_io/.DS_Store","hash":"df2fbeb1400acda0909a32c1cf6bf492f1121e07","modified":1727583174885},{"_id":"public/2024/09/30/MapReduce/index.html","hash":"df37ca83b3f93542824eecb2c11a504392506b67","modified":1727727207472},{"_id":"public/img/yz_favicon.png","hash":"76b5f9b679da101aa92a32176115a52c1cda5e32","modified":1727725677032},{"_id":"public/img/favicon_io/apple-touch-icon.png","hash":"220337f927628b442b446690dc13b8d454a9aa7a","modified":1727725677032},{"_id":"public/img/favicon_io/favicon-16x16.png","hash":"3dd88a7c812be5cd897f810e32bdf55f7d872867","modified":1727725677032},{"_id":"public/img/favicon_io/favicon-32x32.png","hash":"ca31fa2f8dd242981e5a4058ab069be038d1b275","modified":1727725677032},{"_id":"public/img/favicon_io/favicon.ico","hash":"3a7ed482c1d39e241e3b6923cf9c249e44d0b151","modified":1727725677032},{"_id":"public/img/favicon_io/site.webmanifest","hash":"bf31baf91bdd2fcde24a45e3f2a1be33733c6f69","modified":1727725677032},{"_id":"public/img/favicon_io/about.txt","hash":"d30c209db7ea73e07c38a05058382fcb33cf6df0","modified":1727725677032},{"_id":"public/img/favicon_io/android-chrome-512x512.png","hash":"9602d6ace9a9b038addbf53e2c7a876b9a4a2abb","modified":1727725677032},{"_id":"public/categories/Realtime-Big-Data-Analyic/index.html","hash":"6918c395d4c6c3614e2a9288e1a5a8e11c49baf1","modified":1727727207472},{"_id":"source/img/.DS_Store","hash":"9c1202001689743723d54aed6e088cdfdcf9fd3f","modified":1727902014919},{"_id":"source/img/MapReduce/mapreduce_workflow.png","hash":"e94291452e6cd50b1c477ae12008ce5792a73e4c","modified":1727728955779},{"_id":"source/img/MapReduce/mapreduce_example.png","hash":"b87723d2a3bf229003e7a8fb9259076100f92a09","modified":1727732444958},{"_id":"themes/fluid/source/img/background.jpg","hash":"bc671451b6b57fa4db54e69b6040716529dd2ae3","modified":1693859194234},{"_id":"source/img/MapReduce/datalocality.png","hash":"364d42665a09f215aca748f7ae787197d2aabdd1","modified":1727901624913},{"_id":"source/img/MapReduce/yarnlayer.png","hash":"03fc5fc154f12bad133c1e6789df49e5ac61a0e4","modified":1727912184599},{"_id":"source/img/MapReduce/yarnapp.png","hash":"9d7d8d906ecfc87e7f70ec86643e586b2a2b8558","modified":1727914662266},{"_id":"source/img/MapReduce/shufflesort.png","hash":"b5f8a44e1a4981b2bc1c062e94877e3d0c786ad7","modified":1727983541377},{"_id":"source/img/MapReduce/mapreduce_jobexec.png","hash":"8d7999fa5e068322895d6f5647f9d0a2ac4995fd","modified":1727917136876},{"_id":"source/_posts/MapReduce-Programming.md","hash":"150a8726085cb5931dff168f3fa452e0a735309d","modified":1727988194620}],"Category":[{"name":"Realtime Big Data Analysis","_id":"cm1pftsb20000u5iu9t080ypg"},{"name":"Realtime Big Data Analyic","_id":"cm1pg65gf00006kiudhxkae3y"},{"name":"Realtime Big Data Analytic","_id":"cm1tpd84m0000h5ivcy7fa1yq"},{"name":"Realtime and Big Data Analytics","_id":"cm1tpffqr0002h5iv3v7q9v72"}],"Data":[],"Page":[{"title":"VectorZYJ","layout":"about","comment":"valine","date":"2024-09-29T00:59:40.000Z","_content":"\n欢迎来到VectorZYJ的博客，该博客基于[Hexo](https://hexo.io/zh-cn/docs)搭建，使用的主题为[Fluid](https://hexo.fluid-dev.com/docs)","source":"about/index.md","raw":"---\ntitle: VectorZYJ\nlayout: about\ncomment: 'valine'\ndate: 2024-09-28 20:59:40\n---\n\n欢迎来到VectorZYJ的博客，该博客基于[Hexo](https://hexo.io/zh-cn/docs)搭建，使用的主题为[Fluid](https://hexo.fluid-dev.com/docs)","updated":"2024-09-29T02:30:37.927Z","path":"about/index.html","_id":"cm1mvtfg60000ob0c4row9uno","comments":1,"content":"<p>欢迎来到VectorZYJ的博客，该博客基于<a href=\"https://hexo.io/zh-cn/docs\">Hexo</a>搭建，使用的主题为<a href=\"https://hexo.fluid-dev.com/docs\">Fluid</a></p>\n","excerpt":"","more":"<p>欢迎来到VectorZYJ的博客，该博客基于<a href=\"https://hexo.io/zh-cn/docs\">Hexo</a>搭建，使用的主题为<a href=\"https://hexo.fluid-dev.com/docs\">Fluid</a></p>\n"}],"Post":[{"title":"Hello World","_content":"Welcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","source":"_posts/hello-world.md","raw":"---\ntitle: Hello World\n---\nWelcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","slug":"hello-world","published":1,"date":"2024-09-29T00:29:52.739Z","updated":"2024-09-29T00:29:52.740Z","comments":1,"layout":"post","photos":[],"_id":"cm1mujjxi0000450cc5wdbqej","content":"<p>Welcome to <a href=\"https://hexo.io/\">Hexo</a>! This is your very first post. Check <a href=\"https://hexo.io/docs/\">documentation</a> for more info. If you get any problems when using Hexo, you can find the answer in <a href=\"https://hexo.io/docs/troubleshooting.html\">troubleshooting</a> or you can ask me on <a href=\"https://github.com/hexojs/hexo/issues\">GitHub</a>.</p>\n<h2 id=\"Quick-Start\"><a href=\"#Quick-Start\" class=\"headerlink\" title=\"Quick Start\"></a>Quick Start</h2><h3 id=\"Create-a-new-post\"><a href=\"#Create-a-new-post\" class=\"headerlink\" title=\"Create a new post\"></a>Create a new post</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo new <span class=\"string\">"My New Post"</span></span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/writing.html\">Writing</a></p>\n<h3 id=\"Run-server\"><a href=\"#Run-server\" class=\"headerlink\" title=\"Run server\"></a>Run server</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo server</span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/server.html\">Server</a></p>\n<h3 id=\"Generate-static-files\"><a href=\"#Generate-static-files\" class=\"headerlink\" title=\"Generate static files\"></a>Generate static files</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo generate</span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/generating.html\">Generating</a></p>\n<h3 id=\"Deploy-to-remote-sites\"><a href=\"#Deploy-to-remote-sites\" class=\"headerlink\" title=\"Deploy to remote sites\"></a>Deploy to remote sites</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo deploy</span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/one-command-deployment.html\">Deployment</a></p>\n","excerpt":"","more":"<p>Welcome to <a href=\"https://hexo.io/\">Hexo</a>! This is your very first post. Check <a href=\"https://hexo.io/docs/\">documentation</a> for more info. If you get any problems when using Hexo, you can find the answer in <a href=\"https://hexo.io/docs/troubleshooting.html\">troubleshooting</a> or you can ask me on <a href=\"https://github.com/hexojs/hexo/issues\">GitHub</a>.</p>\n<h2 id=\"Quick-Start\"><a href=\"#Quick-Start\" class=\"headerlink\" title=\"Quick Start\"></a>Quick Start</h2><h3 id=\"Create-a-new-post\"><a href=\"#Create-a-new-post\" class=\"headerlink\" title=\"Create a new post\"></a>Create a new post</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo new <span class=\"string\">"My New Post"</span></span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/writing.html\">Writing</a></p>\n<h3 id=\"Run-server\"><a href=\"#Run-server\" class=\"headerlink\" title=\"Run server\"></a>Run server</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo server</span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/server.html\">Server</a></p>\n<h3 id=\"Generate-static-files\"><a href=\"#Generate-static-files\" class=\"headerlink\" title=\"Generate static files\"></a>Generate static files</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo generate</span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/generating.html\">Generating</a></p>\n<h3 id=\"Deploy-to-remote-sites\"><a href=\"#Deploy-to-remote-sites\" class=\"headerlink\" title=\"Deploy to remote sites\"></a>Deploy to remote sites</h3><figure class=\"highlight bash\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">$ hexo deploy</span><br></pre></td></tr></table></figure>\n\n<p>More info: <a href=\"https://hexo.io/docs/one-command-deployment.html\">Deployment</a></p>\n"},{"title":"test","date":"2024-09-29T00:33:43.000Z","_content":"\n### This is a test file\n\n`gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f`\n\n`spark-shell --deploy-mode client`\n# Lab1\n```\ngcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f\nhadoop fs -ls /\nhadoop fs -ls\nhadoop fs -ls /user/yz10317_nyu_edu\nhadoop fs -mkdir lab1\nhadoop fs -ls\nhadoop fs -getfacl lab1\nwget https: //www.gutenberg.org/files/1661/1661-0.txt\nhadoop fs -put 1661-0.txt lab1/data.txt\nhadoop fs -ls lab1\nhadoop fs -get lab1/data.txt\ndiff 1661-0.txt data.txt\nhadoop fs -rm -r lab1\nhadoop fs -ls\nspark-shell --deploy-mode client\n```\n```Scala\n:help\nsc.\\tab\nsc.version\nval myConstant: Int = 2437\nmyConstant\nmyConstant.\\tab\nmyConstant.to\\tab\nmyConstant.toFloat\nmyConstant\nmyConstant.toFloat.toInt\nval myString = myConstant.toString\n:type myString\n:q\n```\n# Lab 2\n```\ngcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f\nspark-shell --deploy-mode client\n```\n```Scala\nval exchangeRate: Double = 1.04\nval dollars: Int = 100.00\nval dollars: Double = 100.00\nvar euros = 0.00\neuros = dollars * exchangeRate\ndollars = 500\nval dollars = 500\nvar eurosInt: Int = 0\neurosInt = dollars * exchangeRate\neurosInt = (dollars * exchangeRate).toInt\nexchangeRate.getClass\ndollars.getClass\neuros.getClass\neurosInt.getClass\nprintln(f\"September 2023: $$$dollars = $eurosInt euros\")\n\n\nval record = \"2023-09-25:19:10:00, 12345678-aaaa-1000-gggg-000111222333, 58, TRUE, enabled, disabled, 37.819722,-122.478611\"\nrecord.length\nrecord.contains(\"disabled\")\nrecord.indexOf(',')\nrecord.toLowerCase.indexOf(\"true\")\nrecord\nvar record2 = record\nrecord == record2\nrecord2 = \"something else\"\nrecord == record2\n```\n```\nhdfs dfs -mkdir loudacre\nhdfs dfs -mkdir loudacre/weblog\n//download 2014-03-15.log and upload it to nyu-dataproc-hdfs-ingest\nhadoop distcp gs://nyu-dataproc-hdfs-ingest/2014-03-15.log loudacre/weblog\nspark-shell --deploy-mode client\n```\n```Scala\nval logfile: String = \"/user/yz10317_nyu_edu/loudacre/weblog/2014-03-15.log\"\nval originalRdd = sc.textFile(logfile)\noriginalRdd.take(5)\nval jpgRdd = originalRdd.filter(line => line.contains(\"jpg\"))\njpgRdd.take(5)\noriginalRdd.filter(line => line.contains(\"jpg\")).count()\nval ipRdd = originalRdd.map(line => line.substring(0, line.indexOf('-')-1))\nipRdd.saveAsTextFile(\"loudacre/iplist\")\n:q\n```\n```\nhdfs dfs -ls loudacre/iplist\nhdfs dfs -cat loudacre/iplist/part-00000\n```\n\n# Lab 3\n```Scala\nval data = sc.textFile(\"loudacre/devstatus/devicestatus.txt\")\nval splitedData = data.map(line => line.split(line(19)))\nval filteredData = splitedData.filter(line => line.length == 14)\nval extractedData = filteredData.map(line => Array(line(0), line(1), line(2), line(12), line(13)))\ndef modify(line: Array[String]) = {\n\tline(1) = line(1).split(' ')(0)\n\tline\n}\nval modifiedData = extractedData.map(modify)\nval finalData = modifiedData.map(line => line.mkString(\",\"))\nfinalData.saveAsTextFile(\"loudacre/devstatus/devicestatus_etl\")\n```\n\n# Lab 4\n```Scala\nval requestCountsRdd = (\n\tsc.textFile(\"loudacre/weblog/2014-03-15.log\")\n\t.map(_.split(' ')(2).toInt -> 1)\n\t.reduceByKey(_ + _)\n\t.sortByKey()\n)\n\nrequestCountsRdd.take(10)\n\nval visitFrequencyRdd = (\n\tsc.textFile(\"loudacre/weblog/2014-03-15.log\")\n\t.map(_.split(' ')(2).toInt -> 1)\n\t.reduceByKey(_ + _)\n\t.map(_._2 -> 1)\n\t.reduceByKey(_ + _)\n\t.sortByKey()\n)\n\nvisitFrequencyRdd.collect()\n\nval userIpList = (\n\tsc.textFile(\"loudacre/weblog/2014-03-15.log\", 4)\n\t.map(line => line.split(' ')(2).toInt -> line.split(' ')(0))\n\t.distinct()\n\t.groupByKey()\n\t.sortByKey()\n\t.mapValues(_.toArray)\n\t.map(tuple => tuple._1.toString + \": \" + tuple._2.mkString(\" \"))\n)\n\nuserIpList.saveAsTextFile(\"loudacre/useriplist\")\n\n:q\n```\n```\nhdfs dfs -ls loudacre/useriplist\nhdfs dfs -cat loudacre/useriplist/part-00000 | head -5\nhdfs dfs -get loudacre/useriplist .\n\n./google-cloud-sdk/bin/gcloud init\ngcloud compute scp --recurse nyu-dataproc-m:~/useriplist ~\n```\n\n# Lab 5\n```Scala\nimport scala.xml._\n\nval resultRdd = {\n\tsc.wholeTextFiles(\"loudacre/activations/*.xml\") // RDD[(String, String)]\n\t.map(x => XML.loadString(x._2) \\ \"activation\") // RDD[scala.xml.NodeSeq]\n\t.flatMap(x => x) // RDD[scala.xml.Node]\n\t.map(x => (x \\ \"account-number\").text -> (x \\ \"model\").text)\n\t.map(x => x._1.toString + \":\" + x._2.toString)\n}\n\nresultRdd.saveAsTextFile(\"loudacre/account-models\")\nspark-submit --master yarn --class AccountModel lab5/target/scala-2.12/accountmodel_2.12-1.0.0.jar\n```\n\n# Lab 6\n```\nhadoop distcp gs://nyu-dataproc-hdfs-ingest/iot_devices.json /user/yz10317_nyu_edu\n```\n```Scala\ncase class DeviceIoTData(\n battery_level: Long,\n co2_level: Long,\n cca2: String,\n cca3: String,\n cn: String,\n device_id: Long,\n device_name: String,\n humidity: Long,\n ip: String,\n latitude: Double,\n lcd: String,\n longitude: Double,\n scale: String,\n temp: Long,\n timestamp: Long\n)\nval ds = spark.read.json(\"iot_devices.json\").as[DeviceIoTData]\nds.printSchema\nds.show(5, false)\nds.first()\n\nval failDevices = {\n\tds.select(\"device_id\", \"battery_level\")\n\t.where($\"battery_level\" === 0)\n}\nfailDevices.count()\nfailDevices.show(5, false)\n\nval co2Emission = {\n\tds.select(\"cn\",\"co2_level\")\n\t.groupBy(\"cn\")\n\t.agg(avg(\"co2_level\"))\n\t.withColumnRenamed(\"avg(co2_level)\", \"average_co2_level\")\n\t.sort($\"average_co2_level\".desc)\n}\nco2Emission.show(5, false)\n```\n\n# Lab 7\n```Scala\nval data = Seq((\"Alice\", \"111-11-1111\"), (\"Bob\", \"123-45-6789\"), (\"Carol\", \"987-65-4321\"))\nval df = data.toDF(\"name\", \"ssn\")\ndf.show()\n\ndf.createOrReplaceTempView(\"users\")\nspark.sql(\"SELECT * FROM users\").show()\n\nval hide = (ssn: String) => { \n\t\"***-**-\".concat(ssn.substring(7, 11))\n}\nspark.udf.register(\"hide\", hide)\nspark.sql(\"SELECT name, hide(ssn) As ssn FROM users\").show()\n```\n","source":"_posts/test.md","raw":"---\ntitle: test\ndate: 2024-09-28 20:33:43\ntags:\n---\n\n### This is a test file\n\n`gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f`\n\n`spark-shell --deploy-mode client`\n# Lab1\n```\ngcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f\nhadoop fs -ls /\nhadoop fs -ls\nhadoop fs -ls /user/yz10317_nyu_edu\nhadoop fs -mkdir lab1\nhadoop fs -ls\nhadoop fs -getfacl lab1\nwget https: //www.gutenberg.org/files/1661/1661-0.txt\nhadoop fs -put 1661-0.txt lab1/data.txt\nhadoop fs -ls lab1\nhadoop fs -get lab1/data.txt\ndiff 1661-0.txt data.txt\nhadoop fs -rm -r lab1\nhadoop fs -ls\nspark-shell --deploy-mode client\n```\n```Scala\n:help\nsc.\\tab\nsc.version\nval myConstant: Int = 2437\nmyConstant\nmyConstant.\\tab\nmyConstant.to\\tab\nmyConstant.toFloat\nmyConstant\nmyConstant.toFloat.toInt\nval myString = myConstant.toString\n:type myString\n:q\n```\n# Lab 2\n```\ngcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f\nspark-shell --deploy-mode client\n```\n```Scala\nval exchangeRate: Double = 1.04\nval dollars: Int = 100.00\nval dollars: Double = 100.00\nvar euros = 0.00\neuros = dollars * exchangeRate\ndollars = 500\nval dollars = 500\nvar eurosInt: Int = 0\neurosInt = dollars * exchangeRate\neurosInt = (dollars * exchangeRate).toInt\nexchangeRate.getClass\ndollars.getClass\neuros.getClass\neurosInt.getClass\nprintln(f\"September 2023: $$$dollars = $eurosInt euros\")\n\n\nval record = \"2023-09-25:19:10:00, 12345678-aaaa-1000-gggg-000111222333, 58, TRUE, enabled, disabled, 37.819722,-122.478611\"\nrecord.length\nrecord.contains(\"disabled\")\nrecord.indexOf(',')\nrecord.toLowerCase.indexOf(\"true\")\nrecord\nvar record2 = record\nrecord == record2\nrecord2 = \"something else\"\nrecord == record2\n```\n```\nhdfs dfs -mkdir loudacre\nhdfs dfs -mkdir loudacre/weblog\n//download 2014-03-15.log and upload it to nyu-dataproc-hdfs-ingest\nhadoop distcp gs://nyu-dataproc-hdfs-ingest/2014-03-15.log loudacre/weblog\nspark-shell --deploy-mode client\n```\n```Scala\nval logfile: String = \"/user/yz10317_nyu_edu/loudacre/weblog/2014-03-15.log\"\nval originalRdd = sc.textFile(logfile)\noriginalRdd.take(5)\nval jpgRdd = originalRdd.filter(line => line.contains(\"jpg\"))\njpgRdd.take(5)\noriginalRdd.filter(line => line.contains(\"jpg\")).count()\nval ipRdd = originalRdd.map(line => line.substring(0, line.indexOf('-')-1))\nipRdd.saveAsTextFile(\"loudacre/iplist\")\n:q\n```\n```\nhdfs dfs -ls loudacre/iplist\nhdfs dfs -cat loudacre/iplist/part-00000\n```\n\n# Lab 3\n```Scala\nval data = sc.textFile(\"loudacre/devstatus/devicestatus.txt\")\nval splitedData = data.map(line => line.split(line(19)))\nval filteredData = splitedData.filter(line => line.length == 14)\nval extractedData = filteredData.map(line => Array(line(0), line(1), line(2), line(12), line(13)))\ndef modify(line: Array[String]) = {\n\tline(1) = line(1).split(' ')(0)\n\tline\n}\nval modifiedData = extractedData.map(modify)\nval finalData = modifiedData.map(line => line.mkString(\",\"))\nfinalData.saveAsTextFile(\"loudacre/devstatus/devicestatus_etl\")\n```\n\n# Lab 4\n```Scala\nval requestCountsRdd = (\n\tsc.textFile(\"loudacre/weblog/2014-03-15.log\")\n\t.map(_.split(' ')(2).toInt -> 1)\n\t.reduceByKey(_ + _)\n\t.sortByKey()\n)\n\nrequestCountsRdd.take(10)\n\nval visitFrequencyRdd = (\n\tsc.textFile(\"loudacre/weblog/2014-03-15.log\")\n\t.map(_.split(' ')(2).toInt -> 1)\n\t.reduceByKey(_ + _)\n\t.map(_._2 -> 1)\n\t.reduceByKey(_ + _)\n\t.sortByKey()\n)\n\nvisitFrequencyRdd.collect()\n\nval userIpList = (\n\tsc.textFile(\"loudacre/weblog/2014-03-15.log\", 4)\n\t.map(line => line.split(' ')(2).toInt -> line.split(' ')(0))\n\t.distinct()\n\t.groupByKey()\n\t.sortByKey()\n\t.mapValues(_.toArray)\n\t.map(tuple => tuple._1.toString + \": \" + tuple._2.mkString(\" \"))\n)\n\nuserIpList.saveAsTextFile(\"loudacre/useriplist\")\n\n:q\n```\n```\nhdfs dfs -ls loudacre/useriplist\nhdfs dfs -cat loudacre/useriplist/part-00000 | head -5\nhdfs dfs -get loudacre/useriplist .\n\n./google-cloud-sdk/bin/gcloud init\ngcloud compute scp --recurse nyu-dataproc-m:~/useriplist ~\n```\n\n# Lab 5\n```Scala\nimport scala.xml._\n\nval resultRdd = {\n\tsc.wholeTextFiles(\"loudacre/activations/*.xml\") // RDD[(String, String)]\n\t.map(x => XML.loadString(x._2) \\ \"activation\") // RDD[scala.xml.NodeSeq]\n\t.flatMap(x => x) // RDD[scala.xml.Node]\n\t.map(x => (x \\ \"account-number\").text -> (x \\ \"model\").text)\n\t.map(x => x._1.toString + \":\" + x._2.toString)\n}\n\nresultRdd.saveAsTextFile(\"loudacre/account-models\")\nspark-submit --master yarn --class AccountModel lab5/target/scala-2.12/accountmodel_2.12-1.0.0.jar\n```\n\n# Lab 6\n```\nhadoop distcp gs://nyu-dataproc-hdfs-ingest/iot_devices.json /user/yz10317_nyu_edu\n```\n```Scala\ncase class DeviceIoTData(\n battery_level: Long,\n co2_level: Long,\n cca2: String,\n cca3: String,\n cn: String,\n device_id: Long,\n device_name: String,\n humidity: Long,\n ip: String,\n latitude: Double,\n lcd: String,\n longitude: Double,\n scale: String,\n temp: Long,\n timestamp: Long\n)\nval ds = spark.read.json(\"iot_devices.json\").as[DeviceIoTData]\nds.printSchema\nds.show(5, false)\nds.first()\n\nval failDevices = {\n\tds.select(\"device_id\", \"battery_level\")\n\t.where($\"battery_level\" === 0)\n}\nfailDevices.count()\nfailDevices.show(5, false)\n\nval co2Emission = {\n\tds.select(\"cn\",\"co2_level\")\n\t.groupBy(\"cn\")\n\t.agg(avg(\"co2_level\"))\n\t.withColumnRenamed(\"avg(co2_level)\", \"average_co2_level\")\n\t.sort($\"average_co2_level\".desc)\n}\nco2Emission.show(5, false)\n```\n\n# Lab 7\n```Scala\nval data = Seq((\"Alice\", \"111-11-1111\"), (\"Bob\", \"123-45-6789\"), (\"Carol\", \"987-65-4321\"))\nval df = data.toDF(\"name\", \"ssn\")\ndf.show()\n\ndf.createOrReplaceTempView(\"users\")\nspark.sql(\"SELECT * FROM users\").show()\n\nval hide = (ssn: String) => { \n\t\"***-**-\".concat(ssn.substring(7, 11))\n}\nspark.udf.register(\"hide\", hide)\nspark.sql(\"SELECT name, hide(ssn) As ssn FROM users\").show()\n```\n","slug":"test","published":1,"updated":"2024-09-29T00:37:58.956Z","_id":"cm1murr4b0000ef0c703d6kwz","comments":1,"layout":"post","photos":[],"content":"<h3 id=\"This-is-a-test-file\"><a href=\"#This-is-a-test-file\" class=\"headerlink\" title=\"This is a test file\"></a>This is a test file</h3><p><code>gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f</code></p>\n<p><code>spark-shell --deploy-mode client</code></p>\n<h1 id=\"Lab1\"><a href=\"#Lab1\" class=\"headerlink\" title=\"Lab1\"></a>Lab1</h1><figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f</span><br><span class=\"line\">hadoop fs -ls /</span><br><span class=\"line\">hadoop fs -ls</span><br><span class=\"line\">hadoop fs -ls /user/yz10317_nyu_edu</span><br><span class=\"line\">hadoop fs -mkdir lab1</span><br><span class=\"line\">hadoop fs -ls</span><br><span class=\"line\">hadoop fs -getfacl lab1</span><br><span class=\"line\">wget https: //www.gutenberg.org/files/1661/1661-0.txt</span><br><span class=\"line\">hadoop fs -put 1661-0.txt lab1/data.txt</span><br><span class=\"line\">hadoop fs -ls lab1</span><br><span class=\"line\">hadoop fs -get lab1/data.txt</span><br><span class=\"line\">diff 1661-0.txt data.txt</span><br><span class=\"line\">hadoop fs -rm -r lab1</span><br><span class=\"line\">hadoop fs -ls</span><br><span class=\"line\">spark-shell --deploy-mode client</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">:help</span><br><span class=\"line\">sc.\\tab</span><br><span class=\"line\">sc.version</span><br><span class=\"line\"><span class=\"keyword\">val</span> myConstant: <span class=\"type\">Int</span> = <span class=\"number\">2437</span></span><br><span class=\"line\">myConstant</span><br><span class=\"line\">myConstant.\\tab</span><br><span class=\"line\">myConstant.to\\tab</span><br><span class=\"line\">myConstant.toFloat</span><br><span class=\"line\">myConstant</span><br><span class=\"line\">myConstant.toFloat.toInt</span><br><span class=\"line\"><span class=\"keyword\">val</span> myString = myConstant.toString</span><br><span class=\"line\">:<span class=\"class\"><span class=\"keyword\">type</span> <span class=\"title\">myString</span></span></span><br><span class=\"line\">:q</span><br></pre></td></tr></table></figure>\n<h1 id=\"Lab-2\"><a href=\"#Lab-2\" class=\"headerlink\" title=\"Lab 2\"></a>Lab 2</h1><figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f</span><br><span class=\"line\">spark-shell --deploy-mode client</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br><span class=\"line\">16</span><br><span class=\"line\">17</span><br><span class=\"line\">18</span><br><span class=\"line\">19</span><br><span class=\"line\">20</span><br><span class=\"line\">21</span><br><span class=\"line\">22</span><br><span class=\"line\">23</span><br><span class=\"line\">24</span><br><span class=\"line\">25</span><br><span class=\"line\">26</span><br><span class=\"line\">27</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> exchangeRate: <span class=\"type\">Double</span> = <span class=\"number\">1.04</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> dollars: <span class=\"type\">Int</span> = <span class=\"number\">100.00</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> dollars: <span class=\"type\">Double</span> = <span class=\"number\">100.00</span></span><br><span class=\"line\"><span class=\"keyword\">var</span> euros = <span class=\"number\">0.00</span></span><br><span class=\"line\">euros = dollars * exchangeRate</span><br><span class=\"line\">dollars = <span class=\"number\">500</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> dollars = <span class=\"number\">500</span></span><br><span class=\"line\"><span class=\"keyword\">var</span> eurosInt: <span class=\"type\">Int</span> = <span class=\"number\">0</span></span><br><span class=\"line\">eurosInt = dollars * exchangeRate</span><br><span class=\"line\">eurosInt = (dollars * exchangeRate).toInt</span><br><span class=\"line\">exchangeRate.getClass</span><br><span class=\"line\">dollars.getClass</span><br><span class=\"line\">euros.getClass</span><br><span class=\"line\">eurosInt.getClass</span><br><span class=\"line\">println(<span class=\"string\">f"September 2023: $$<span class=\"subst\">$dollars</span> = <span class=\"subst\">$eurosInt</span> euros"</span>)</span><br><span class=\"line\"></span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> record = <span class=\"string\">"2023-09-25:19:10:00, 12345678-aaaa-1000-gggg-000111222333, 58, TRUE, enabled, disabled, 37.819722,-122.478611"</span></span><br><span class=\"line\">record.length</span><br><span class=\"line\">record.contains(<span class=\"string\">"disabled"</span>)</span><br><span class=\"line\">record.indexOf(',')</span><br><span class=\"line\">record.toLowerCase.indexOf(<span class=\"string\">"true"</span>)</span><br><span class=\"line\">record</span><br><span class=\"line\"><span class=\"keyword\">var</span> record2 = record</span><br><span class=\"line\">record == record2</span><br><span class=\"line\">record2 = <span class=\"string\">"something else"</span></span><br><span class=\"line\">record == record2</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hdfs dfs -mkdir loudacre</span><br><span class=\"line\">hdfs dfs -mkdir loudacre/weblog</span><br><span class=\"line\">//download 2014-03-15.log and upload it to nyu-dataproc-hdfs-ingest</span><br><span class=\"line\">hadoop distcp gs://nyu-dataproc-hdfs-ingest/2014-03-15.log loudacre/weblog</span><br><span class=\"line\">spark-shell --deploy-mode client</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> logfile: <span class=\"type\">String</span> = <span class=\"string\">"/user/yz10317_nyu_edu/loudacre/weblog/2014-03-15.log"</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> originalRdd = sc.textFile(logfile)</span><br><span class=\"line\">originalRdd.take(<span class=\"number\">5</span>)</span><br><span class=\"line\"><span class=\"keyword\">val</span> jpgRdd = originalRdd.filter(line => line.contains(<span class=\"string\">"jpg"</span>))</span><br><span class=\"line\">jpgRdd.take(<span class=\"number\">5</span>)</span><br><span class=\"line\">originalRdd.filter(line => line.contains(<span class=\"string\">"jpg"</span>)).count()</span><br><span class=\"line\"><span class=\"keyword\">val</span> ipRdd = originalRdd.map(line => line.substring(<span class=\"number\">0</span>, line.indexOf('-')<span class=\"number\">-1</span>))</span><br><span class=\"line\">ipRdd.saveAsTextFile(<span class=\"string\">"loudacre/iplist"</span>)</span><br><span class=\"line\">:q</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hdfs dfs -ls loudacre/iplist</span><br><span class=\"line\">hdfs dfs -cat loudacre/iplist/part-00000</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-3\"><a href=\"#Lab-3\" class=\"headerlink\" title=\"Lab 3\"></a>Lab 3</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> data = sc.textFile(<span class=\"string\">"loudacre/devstatus/devicestatus.txt"</span>)</span><br><span class=\"line\"><span class=\"keyword\">val</span> splitedData = data.map(line => line.split(line(<span class=\"number\">19</span>)))</span><br><span class=\"line\"><span class=\"keyword\">val</span> filteredData = splitedData.filter(line => line.length == <span class=\"number\">14</span>)</span><br><span class=\"line\"><span class=\"keyword\">val</span> extractedData = filteredData.map(line => <span class=\"type\">Array</span>(line(<span class=\"number\">0</span>), line(<span class=\"number\">1</span>), line(<span class=\"number\">2</span>), line(<span class=\"number\">12</span>), line(<span class=\"number\">13</span>)))</span><br><span class=\"line\"><span class=\"function\"><span class=\"keyword\">def</span> <span class=\"title\">modify</span></span>(line: <span class=\"type\">Array</span>[<span class=\"type\">String</span>]) = {</span><br><span class=\"line\">\tline(<span class=\"number\">1</span>) = line(<span class=\"number\">1</span>).split(' ')(<span class=\"number\">0</span>)</span><br><span class=\"line\">\tline</span><br><span class=\"line\">}</span><br><span class=\"line\"><span class=\"keyword\">val</span> modifiedData = extractedData.map(modify)</span><br><span class=\"line\"><span class=\"keyword\">val</span> finalData = modifiedData.map(line => line.mkString(<span class=\"string\">","</span>))</span><br><span class=\"line\">finalData.saveAsTextFile(<span class=\"string\">"loudacre/devstatus/devicestatus_etl"</span>)</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-4\"><a href=\"#Lab-4\" class=\"headerlink\" title=\"Lab 4\"></a>Lab 4</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br><span class=\"line\">16</span><br><span class=\"line\">17</span><br><span class=\"line\">18</span><br><span class=\"line\">19</span><br><span class=\"line\">20</span><br><span class=\"line\">21</span><br><span class=\"line\">22</span><br><span class=\"line\">23</span><br><span class=\"line\">24</span><br><span class=\"line\">25</span><br><span class=\"line\">26</span><br><span class=\"line\">27</span><br><span class=\"line\">28</span><br><span class=\"line\">29</span><br><span class=\"line\">30</span><br><span class=\"line\">31</span><br><span class=\"line\">32</span><br><span class=\"line\">33</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> requestCountsRdd = (</span><br><span class=\"line\">\tsc.textFile(<span class=\"string\">"loudacre/weblog/2014-03-15.log"</span>)</span><br><span class=\"line\">\t.map(_.split(' ')(<span class=\"number\">2</span>).toInt -> <span class=\"number\">1</span>)</span><br><span class=\"line\">\t.reduceByKey(_ + _)</span><br><span class=\"line\">\t.sortByKey()</span><br><span class=\"line\">)</span><br><span class=\"line\"></span><br><span class=\"line\">requestCountsRdd.take(<span class=\"number\">10</span>)</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> visitFrequencyRdd = (</span><br><span class=\"line\">\tsc.textFile(<span class=\"string\">"loudacre/weblog/2014-03-15.log"</span>)</span><br><span class=\"line\">\t.map(_.split(' ')(<span class=\"number\">2</span>).toInt -> <span class=\"number\">1</span>)</span><br><span class=\"line\">\t.reduceByKey(_ + _)</span><br><span class=\"line\">\t.map(_._2 -> <span class=\"number\">1</span>)</span><br><span class=\"line\">\t.reduceByKey(_ + _)</span><br><span class=\"line\">\t.sortByKey()</span><br><span class=\"line\">)</span><br><span class=\"line\"></span><br><span class=\"line\">visitFrequencyRdd.collect()</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> userIpList = (</span><br><span class=\"line\">\tsc.textFile(<span class=\"string\">"loudacre/weblog/2014-03-15.log"</span>, <span class=\"number\">4</span>)</span><br><span class=\"line\">\t.map(line => line.split(' ')(<span class=\"number\">2</span>).toInt -> line.split(' ')(<span class=\"number\">0</span>))</span><br><span class=\"line\">\t.distinct()</span><br><span class=\"line\">\t.groupByKey()</span><br><span class=\"line\">\t.sortByKey()</span><br><span class=\"line\">\t.mapValues(_.toArray)</span><br><span class=\"line\">\t.map(tuple => tuple._1.toString + <span class=\"string\">": "</span> + tuple._2.mkString(<span class=\"string\">" "</span>))</span><br><span class=\"line\">)</span><br><span class=\"line\"></span><br><span class=\"line\">userIpList.saveAsTextFile(<span class=\"string\">"loudacre/useriplist"</span>)</span><br><span class=\"line\"></span><br><span class=\"line\">:q</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hdfs dfs -ls loudacre/useriplist</span><br><span class=\"line\">hdfs dfs -cat loudacre/useriplist/part-00000 | head -5</span><br><span class=\"line\">hdfs dfs -get loudacre/useriplist .</span><br><span class=\"line\"></span><br><span class=\"line\">./google-cloud-sdk/bin/gcloud init</span><br><span class=\"line\">gcloud compute scp --recurse nyu-dataproc-m:~/useriplist ~</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-5\"><a href=\"#Lab-5\" class=\"headerlink\" title=\"Lab 5\"></a>Lab 5</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">import</span> scala.xml._</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> resultRdd = {</span><br><span class=\"line\">\tsc.wholeTextFiles(<span class=\"string\">"loudacre/activations/*.xml"</span>) <span class=\"comment\">// RDD[(String, String)]</span></span><br><span class=\"line\">\t.map(x => <span class=\"type\">XML</span>.loadString(x._2) \\ <span class=\"string\">"activation"</span>) <span class=\"comment\">// RDD[scala.xml.NodeSeq]</span></span><br><span class=\"line\">\t.flatMap(x => x) <span class=\"comment\">// RDD[scala.xml.Node]</span></span><br><span class=\"line\">\t.map(x => (x \\ <span class=\"string\">"account-number"</span>).text -> (x \\ <span class=\"string\">"model"</span>).text)</span><br><span class=\"line\">\t.map(x => x._1.toString + <span class=\"string\">":"</span> + x._2.toString)</span><br><span class=\"line\">}</span><br><span class=\"line\"></span><br><span class=\"line\">resultRdd.saveAsTextFile(<span class=\"string\">"loudacre/account-models"</span>)</span><br><span class=\"line\">spark-submit --master yarn --<span class=\"class\"><span class=\"keyword\">class</span> <span class=\"title\">AccountModel</span> <span class=\"title\">lab5/target/scala-2</span>.12<span class=\"title\">/accountmodel_2</span>.12<span class=\"title\">-1</span>.0.0.<span class=\"title\">jar</span></span></span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-6\"><a href=\"#Lab-6\" class=\"headerlink\" title=\"Lab 6\"></a>Lab 6</h1><figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hadoop distcp gs://nyu-dataproc-hdfs-ingest/iot_devices.json /user/yz10317_nyu_edu</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br><span class=\"line\">16</span><br><span class=\"line\">17</span><br><span class=\"line\">18</span><br><span class=\"line\">19</span><br><span class=\"line\">20</span><br><span class=\"line\">21</span><br><span class=\"line\">22</span><br><span class=\"line\">23</span><br><span class=\"line\">24</span><br><span class=\"line\">25</span><br><span class=\"line\">26</span><br><span class=\"line\">27</span><br><span class=\"line\">28</span><br><span class=\"line\">29</span><br><span class=\"line\">30</span><br><span class=\"line\">31</span><br><span class=\"line\">32</span><br><span class=\"line\">33</span><br><span class=\"line\">34</span><br><span class=\"line\">35</span><br><span class=\"line\">36</span><br><span class=\"line\">37</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">case</span> <span class=\"class\"><span class=\"keyword\">class</span> <span class=\"title\">DeviceIoTData</span>(<span class=\"params\"></span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> battery_level: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> co2_level: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> cca2: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> cca3: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> cn: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> device_id: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> device_name: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> humidity: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> ip: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> latitude: <span class=\"type\">Double</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> lcd: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> longitude: <span class=\"type\">Double</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> scale: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> temp: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> timestamp: <span class=\"type\">Long</span></span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"></span>)</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> ds = spark.read.json(<span class=\"string\">"iot_devices.json"</span>).as[<span class=\"type\">DeviceIoTData</span>]</span><br><span class=\"line\">ds.printSchema</span><br><span class=\"line\">ds.show(<span class=\"number\">5</span>, <span class=\"literal\">false</span>)</span><br><span class=\"line\">ds.first()</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> failDevices = {</span><br><span class=\"line\">\tds.select(<span class=\"string\">"device_id"</span>, <span class=\"string\">"battery_level"</span>)</span><br><span class=\"line\">\t.where($<span class=\"string\">"battery_level"</span> === <span class=\"number\">0</span>)</span><br><span class=\"line\">}</span><br><span class=\"line\">failDevices.count()</span><br><span class=\"line\">failDevices.show(<span class=\"number\">5</span>, <span class=\"literal\">false</span>)</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> co2Emission = {</span><br><span class=\"line\">\tds.select(<span class=\"string\">"cn"</span>,<span class=\"string\">"co2_level"</span>)</span><br><span class=\"line\">\t.groupBy(<span class=\"string\">"cn"</span>)</span><br><span class=\"line\">\t.agg(avg(<span class=\"string\">"co2_level"</span>))</span><br><span class=\"line\">\t.withColumnRenamed(<span class=\"string\">"avg(co2_level)"</span>, <span class=\"string\">"average_co2_level"</span>)</span><br><span class=\"line\">\t.sort($<span class=\"string\">"average_co2_level"</span>.desc)</span><br><span class=\"line\">}</span><br><span class=\"line\">co2Emission.show(<span class=\"number\">5</span>, <span class=\"literal\">false</span>)</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-7\"><a href=\"#Lab-7\" class=\"headerlink\" title=\"Lab 7\"></a>Lab 7</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> data = <span class=\"type\">Seq</span>((<span class=\"string\">"Alice"</span>, <span class=\"string\">"111-11-1111"</span>), (<span class=\"string\">"Bob"</span>, <span class=\"string\">"123-45-6789"</span>), (<span class=\"string\">"Carol"</span>, <span class=\"string\">"987-65-4321"</span>))</span><br><span class=\"line\"><span class=\"keyword\">val</span> df = data.toDF(<span class=\"string\">"name"</span>, <span class=\"string\">"ssn"</span>)</span><br><span class=\"line\">df.show()</span><br><span class=\"line\"></span><br><span class=\"line\">df.createOrReplaceTempView(<span class=\"string\">"users"</span>)</span><br><span class=\"line\">spark.sql(<span class=\"string\">"SELECT * FROM users"</span>).show()</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> hide = (ssn: <span class=\"type\">String</span>) => { </span><br><span class=\"line\">\t<span class=\"string\">"***-**-"</span>.concat(ssn.substring(<span class=\"number\">7</span>, <span class=\"number\">11</span>))</span><br><span class=\"line\">}</span><br><span class=\"line\">spark.udf.register(<span class=\"string\">"hide"</span>, hide)</span><br><span class=\"line\">spark.sql(<span class=\"string\">"SELECT name, hide(ssn) As ssn FROM users"</span>).show()</span><br></pre></td></tr></table></figure>\n","excerpt":"","more":"<h3 id=\"This-is-a-test-file\"><a href=\"#This-is-a-test-file\" class=\"headerlink\" title=\"This is a test file\"></a>This is a test file</h3><p><code>gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f</code></p>\n<p><code>spark-shell --deploy-mode client</code></p>\n<h1 id=\"Lab1\"><a href=\"#Lab1\" class=\"headerlink\" title=\"Lab1\"></a>Lab1</h1><figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f</span><br><span class=\"line\">hadoop fs -ls /</span><br><span class=\"line\">hadoop fs -ls</span><br><span class=\"line\">hadoop fs -ls /user/yz10317_nyu_edu</span><br><span class=\"line\">hadoop fs -mkdir lab1</span><br><span class=\"line\">hadoop fs -ls</span><br><span class=\"line\">hadoop fs -getfacl lab1</span><br><span class=\"line\">wget https: //www.gutenberg.org/files/1661/1661-0.txt</span><br><span class=\"line\">hadoop fs -put 1661-0.txt lab1/data.txt</span><br><span class=\"line\">hadoop fs -ls lab1</span><br><span class=\"line\">hadoop fs -get lab1/data.txt</span><br><span class=\"line\">diff 1661-0.txt data.txt</span><br><span class=\"line\">hadoop fs -rm -r lab1</span><br><span class=\"line\">hadoop fs -ls</span><br><span class=\"line\">spark-shell --deploy-mode client</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">:help</span><br><span class=\"line\">sc.\\tab</span><br><span class=\"line\">sc.version</span><br><span class=\"line\"><span class=\"keyword\">val</span> myConstant: <span class=\"type\">Int</span> = <span class=\"number\">2437</span></span><br><span class=\"line\">myConstant</span><br><span class=\"line\">myConstant.\\tab</span><br><span class=\"line\">myConstant.to\\tab</span><br><span class=\"line\">myConstant.toFloat</span><br><span class=\"line\">myConstant</span><br><span class=\"line\">myConstant.toFloat.toInt</span><br><span class=\"line\"><span class=\"keyword\">val</span> myString = myConstant.toString</span><br><span class=\"line\">:<span class=\"class\"><span class=\"keyword\">type</span> <span class=\"title\">myString</span></span></span><br><span class=\"line\">:q</span><br></pre></td></tr></table></figure>\n<h1 id=\"Lab-2\"><a href=\"#Lab-2\" class=\"headerlink\" title=\"Lab 2\"></a>Lab 2</h1><figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">gcloud compute ssh nyu-dataproc-m --project hpc-dataproc-19b8 --zone us-central1-f</span><br><span class=\"line\">spark-shell --deploy-mode client</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br><span class=\"line\">16</span><br><span class=\"line\">17</span><br><span class=\"line\">18</span><br><span class=\"line\">19</span><br><span class=\"line\">20</span><br><span class=\"line\">21</span><br><span class=\"line\">22</span><br><span class=\"line\">23</span><br><span class=\"line\">24</span><br><span class=\"line\">25</span><br><span class=\"line\">26</span><br><span class=\"line\">27</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> exchangeRate: <span class=\"type\">Double</span> = <span class=\"number\">1.04</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> dollars: <span class=\"type\">Int</span> = <span class=\"number\">100.00</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> dollars: <span class=\"type\">Double</span> = <span class=\"number\">100.00</span></span><br><span class=\"line\"><span class=\"keyword\">var</span> euros = <span class=\"number\">0.00</span></span><br><span class=\"line\">euros = dollars * exchangeRate</span><br><span class=\"line\">dollars = <span class=\"number\">500</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> dollars = <span class=\"number\">500</span></span><br><span class=\"line\"><span class=\"keyword\">var</span> eurosInt: <span class=\"type\">Int</span> = <span class=\"number\">0</span></span><br><span class=\"line\">eurosInt = dollars * exchangeRate</span><br><span class=\"line\">eurosInt = (dollars * exchangeRate).toInt</span><br><span class=\"line\">exchangeRate.getClass</span><br><span class=\"line\">dollars.getClass</span><br><span class=\"line\">euros.getClass</span><br><span class=\"line\">eurosInt.getClass</span><br><span class=\"line\">println(<span class=\"string\">f"September 2023: $$<span class=\"subst\">$dollars</span> = <span class=\"subst\">$eurosInt</span> euros"</span>)</span><br><span class=\"line\"></span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> record = <span class=\"string\">"2023-09-25:19:10:00, 12345678-aaaa-1000-gggg-000111222333, 58, TRUE, enabled, disabled, 37.819722,-122.478611"</span></span><br><span class=\"line\">record.length</span><br><span class=\"line\">record.contains(<span class=\"string\">"disabled"</span>)</span><br><span class=\"line\">record.indexOf(',')</span><br><span class=\"line\">record.toLowerCase.indexOf(<span class=\"string\">"true"</span>)</span><br><span class=\"line\">record</span><br><span class=\"line\"><span class=\"keyword\">var</span> record2 = record</span><br><span class=\"line\">record == record2</span><br><span class=\"line\">record2 = <span class=\"string\">"something else"</span></span><br><span class=\"line\">record == record2</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hdfs dfs -mkdir loudacre</span><br><span class=\"line\">hdfs dfs -mkdir loudacre/weblog</span><br><span class=\"line\">//download 2014-03-15.log and upload it to nyu-dataproc-hdfs-ingest</span><br><span class=\"line\">hadoop distcp gs://nyu-dataproc-hdfs-ingest/2014-03-15.log loudacre/weblog</span><br><span class=\"line\">spark-shell --deploy-mode client</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> logfile: <span class=\"type\">String</span> = <span class=\"string\">"/user/yz10317_nyu_edu/loudacre/weblog/2014-03-15.log"</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> originalRdd = sc.textFile(logfile)</span><br><span class=\"line\">originalRdd.take(<span class=\"number\">5</span>)</span><br><span class=\"line\"><span class=\"keyword\">val</span> jpgRdd = originalRdd.filter(line => line.contains(<span class=\"string\">"jpg"</span>))</span><br><span class=\"line\">jpgRdd.take(<span class=\"number\">5</span>)</span><br><span class=\"line\">originalRdd.filter(line => line.contains(<span class=\"string\">"jpg"</span>)).count()</span><br><span class=\"line\"><span class=\"keyword\">val</span> ipRdd = originalRdd.map(line => line.substring(<span class=\"number\">0</span>, line.indexOf('-')<span class=\"number\">-1</span>))</span><br><span class=\"line\">ipRdd.saveAsTextFile(<span class=\"string\">"loudacre/iplist"</span>)</span><br><span class=\"line\">:q</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hdfs dfs -ls loudacre/iplist</span><br><span class=\"line\">hdfs dfs -cat loudacre/iplist/part-00000</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-3\"><a href=\"#Lab-3\" class=\"headerlink\" title=\"Lab 3\"></a>Lab 3</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> data = sc.textFile(<span class=\"string\">"loudacre/devstatus/devicestatus.txt"</span>)</span><br><span class=\"line\"><span class=\"keyword\">val</span> splitedData = data.map(line => line.split(line(<span class=\"number\">19</span>)))</span><br><span class=\"line\"><span class=\"keyword\">val</span> filteredData = splitedData.filter(line => line.length == <span class=\"number\">14</span>)</span><br><span class=\"line\"><span class=\"keyword\">val</span> extractedData = filteredData.map(line => <span class=\"type\">Array</span>(line(<span class=\"number\">0</span>), line(<span class=\"number\">1</span>), line(<span class=\"number\">2</span>), line(<span class=\"number\">12</span>), line(<span class=\"number\">13</span>)))</span><br><span class=\"line\"><span class=\"function\"><span class=\"keyword\">def</span> <span class=\"title\">modify</span></span>(line: <span class=\"type\">Array</span>[<span class=\"type\">String</span>]) = {</span><br><span class=\"line\">\tline(<span class=\"number\">1</span>) = line(<span class=\"number\">1</span>).split(' ')(<span class=\"number\">0</span>)</span><br><span class=\"line\">\tline</span><br><span class=\"line\">}</span><br><span class=\"line\"><span class=\"keyword\">val</span> modifiedData = extractedData.map(modify)</span><br><span class=\"line\"><span class=\"keyword\">val</span> finalData = modifiedData.map(line => line.mkString(<span class=\"string\">","</span>))</span><br><span class=\"line\">finalData.saveAsTextFile(<span class=\"string\">"loudacre/devstatus/devicestatus_etl"</span>)</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-4\"><a href=\"#Lab-4\" class=\"headerlink\" title=\"Lab 4\"></a>Lab 4</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br><span class=\"line\">16</span><br><span class=\"line\">17</span><br><span class=\"line\">18</span><br><span class=\"line\">19</span><br><span class=\"line\">20</span><br><span class=\"line\">21</span><br><span class=\"line\">22</span><br><span class=\"line\">23</span><br><span class=\"line\">24</span><br><span class=\"line\">25</span><br><span class=\"line\">26</span><br><span class=\"line\">27</span><br><span class=\"line\">28</span><br><span class=\"line\">29</span><br><span class=\"line\">30</span><br><span class=\"line\">31</span><br><span class=\"line\">32</span><br><span class=\"line\">33</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> requestCountsRdd = (</span><br><span class=\"line\">\tsc.textFile(<span class=\"string\">"loudacre/weblog/2014-03-15.log"</span>)</span><br><span class=\"line\">\t.map(_.split(' ')(<span class=\"number\">2</span>).toInt -> <span class=\"number\">1</span>)</span><br><span class=\"line\">\t.reduceByKey(_ + _)</span><br><span class=\"line\">\t.sortByKey()</span><br><span class=\"line\">)</span><br><span class=\"line\"></span><br><span class=\"line\">requestCountsRdd.take(<span class=\"number\">10</span>)</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> visitFrequencyRdd = (</span><br><span class=\"line\">\tsc.textFile(<span class=\"string\">"loudacre/weblog/2014-03-15.log"</span>)</span><br><span class=\"line\">\t.map(_.split(' ')(<span class=\"number\">2</span>).toInt -> <span class=\"number\">1</span>)</span><br><span class=\"line\">\t.reduceByKey(_ + _)</span><br><span class=\"line\">\t.map(_._2 -> <span class=\"number\">1</span>)</span><br><span class=\"line\">\t.reduceByKey(_ + _)</span><br><span class=\"line\">\t.sortByKey()</span><br><span class=\"line\">)</span><br><span class=\"line\"></span><br><span class=\"line\">visitFrequencyRdd.collect()</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> userIpList = (</span><br><span class=\"line\">\tsc.textFile(<span class=\"string\">"loudacre/weblog/2014-03-15.log"</span>, <span class=\"number\">4</span>)</span><br><span class=\"line\">\t.map(line => line.split(' ')(<span class=\"number\">2</span>).toInt -> line.split(' ')(<span class=\"number\">0</span>))</span><br><span class=\"line\">\t.distinct()</span><br><span class=\"line\">\t.groupByKey()</span><br><span class=\"line\">\t.sortByKey()</span><br><span class=\"line\">\t.mapValues(_.toArray)</span><br><span class=\"line\">\t.map(tuple => tuple._1.toString + <span class=\"string\">": "</span> + tuple._2.mkString(<span class=\"string\">" "</span>))</span><br><span class=\"line\">)</span><br><span class=\"line\"></span><br><span class=\"line\">userIpList.saveAsTextFile(<span class=\"string\">"loudacre/useriplist"</span>)</span><br><span class=\"line\"></span><br><span class=\"line\">:q</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hdfs dfs -ls loudacre/useriplist</span><br><span class=\"line\">hdfs dfs -cat loudacre/useriplist/part-00000 | head -5</span><br><span class=\"line\">hdfs dfs -get loudacre/useriplist .</span><br><span class=\"line\"></span><br><span class=\"line\">./google-cloud-sdk/bin/gcloud init</span><br><span class=\"line\">gcloud compute scp --recurse nyu-dataproc-m:~/useriplist ~</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-5\"><a href=\"#Lab-5\" class=\"headerlink\" title=\"Lab 5\"></a>Lab 5</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">import</span> scala.xml._</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> resultRdd = {</span><br><span class=\"line\">\tsc.wholeTextFiles(<span class=\"string\">"loudacre/activations/*.xml"</span>) <span class=\"comment\">// RDD[(String, String)]</span></span><br><span class=\"line\">\t.map(x => <span class=\"type\">XML</span>.loadString(x._2) \\ <span class=\"string\">"activation"</span>) <span class=\"comment\">// RDD[scala.xml.NodeSeq]</span></span><br><span class=\"line\">\t.flatMap(x => x) <span class=\"comment\">// RDD[scala.xml.Node]</span></span><br><span class=\"line\">\t.map(x => (x \\ <span class=\"string\">"account-number"</span>).text -> (x \\ <span class=\"string\">"model"</span>).text)</span><br><span class=\"line\">\t.map(x => x._1.toString + <span class=\"string\">":"</span> + x._2.toString)</span><br><span class=\"line\">}</span><br><span class=\"line\"></span><br><span class=\"line\">resultRdd.saveAsTextFile(<span class=\"string\">"loudacre/account-models"</span>)</span><br><span class=\"line\">spark-submit --master yarn --<span class=\"class\"><span class=\"keyword\">class</span> <span class=\"title\">AccountModel</span> <span class=\"title\">lab5/target/scala-2</span>.12<span class=\"title\">/accountmodel_2</span>.12<span class=\"title\">-1</span>.0.0.<span class=\"title\">jar</span></span></span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-6\"><a href=\"#Lab-6\" class=\"headerlink\" title=\"Lab 6\"></a>Lab 6</h1><figure class=\"highlight plaintext\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br></pre></td><td class=\"code\"><pre><span class=\"line\">hadoop distcp gs://nyu-dataproc-hdfs-ingest/iot_devices.json /user/yz10317_nyu_edu</span><br></pre></td></tr></table></figure>\n<figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br><span class=\"line\">13</span><br><span class=\"line\">14</span><br><span class=\"line\">15</span><br><span class=\"line\">16</span><br><span class=\"line\">17</span><br><span class=\"line\">18</span><br><span class=\"line\">19</span><br><span class=\"line\">20</span><br><span class=\"line\">21</span><br><span class=\"line\">22</span><br><span class=\"line\">23</span><br><span class=\"line\">24</span><br><span class=\"line\">25</span><br><span class=\"line\">26</span><br><span class=\"line\">27</span><br><span class=\"line\">28</span><br><span class=\"line\">29</span><br><span class=\"line\">30</span><br><span class=\"line\">31</span><br><span class=\"line\">32</span><br><span class=\"line\">33</span><br><span class=\"line\">34</span><br><span class=\"line\">35</span><br><span class=\"line\">36</span><br><span class=\"line\">37</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">case</span> <span class=\"class\"><span class=\"keyword\">class</span> <span class=\"title\">DeviceIoTData</span>(<span class=\"params\"></span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> battery_level: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> co2_level: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> cca2: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> cca3: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> cn: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> device_id: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> device_name: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> humidity: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> ip: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> latitude: <span class=\"type\">Double</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> lcd: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> longitude: <span class=\"type\">Double</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> scale: <span class=\"type\">String</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> temp: <span class=\"type\">Long</span>,</span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"> timestamp: <span class=\"type\">Long</span></span></span></span><br><span class=\"line\"><span class=\"params\"><span class=\"class\"></span>)</span></span><br><span class=\"line\"><span class=\"keyword\">val</span> ds = spark.read.json(<span class=\"string\">"iot_devices.json"</span>).as[<span class=\"type\">DeviceIoTData</span>]</span><br><span class=\"line\">ds.printSchema</span><br><span class=\"line\">ds.show(<span class=\"number\">5</span>, <span class=\"literal\">false</span>)</span><br><span class=\"line\">ds.first()</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> failDevices = {</span><br><span class=\"line\">\tds.select(<span class=\"string\">"device_id"</span>, <span class=\"string\">"battery_level"</span>)</span><br><span class=\"line\">\t.where($<span class=\"string\">"battery_level"</span> === <span class=\"number\">0</span>)</span><br><span class=\"line\">}</span><br><span class=\"line\">failDevices.count()</span><br><span class=\"line\">failDevices.show(<span class=\"number\">5</span>, <span class=\"literal\">false</span>)</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> co2Emission = {</span><br><span class=\"line\">\tds.select(<span class=\"string\">"cn"</span>,<span class=\"string\">"co2_level"</span>)</span><br><span class=\"line\">\t.groupBy(<span class=\"string\">"cn"</span>)</span><br><span class=\"line\">\t.agg(avg(<span class=\"string\">"co2_level"</span>))</span><br><span class=\"line\">\t.withColumnRenamed(<span class=\"string\">"avg(co2_level)"</span>, <span class=\"string\">"average_co2_level"</span>)</span><br><span class=\"line\">\t.sort($<span class=\"string\">"average_co2_level"</span>.desc)</span><br><span class=\"line\">}</span><br><span class=\"line\">co2Emission.show(<span class=\"number\">5</span>, <span class=\"literal\">false</span>)</span><br></pre></td></tr></table></figure>\n\n<h1 id=\"Lab-7\"><a href=\"#Lab-7\" class=\"headerlink\" title=\"Lab 7\"></a>Lab 7</h1><figure class=\"highlight scala\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br><span class=\"line\">11</span><br><span class=\"line\">12</span><br></pre></td><td class=\"code\"><pre><span class=\"line\"><span class=\"keyword\">val</span> data = <span class=\"type\">Seq</span>((<span class=\"string\">"Alice"</span>, <span class=\"string\">"111-11-1111"</span>), (<span class=\"string\">"Bob"</span>, <span class=\"string\">"123-45-6789"</span>), (<span class=\"string\">"Carol"</span>, <span class=\"string\">"987-65-4321"</span>))</span><br><span class=\"line\"><span class=\"keyword\">val</span> df = data.toDF(<span class=\"string\">"name"</span>, <span class=\"string\">"ssn"</span>)</span><br><span class=\"line\">df.show()</span><br><span class=\"line\"></span><br><span class=\"line\">df.createOrReplaceTempView(<span class=\"string\">"users"</span>)</span><br><span class=\"line\">spark.sql(<span class=\"string\">"SELECT * FROM users"</span>).show()</span><br><span class=\"line\"></span><br><span class=\"line\"><span class=\"keyword\">val</span> hide = (ssn: <span class=\"type\">String</span>) => { </span><br><span class=\"line\">\t<span class=\"string\">"***-**-"</span>.concat(ssn.substring(<span class=\"number\">7</span>, <span class=\"number\">11</span>))</span><br><span class=\"line\">}</span><br><span class=\"line\">spark.udf.register(<span class=\"string\">"hide"</span>, hide)</span><br><span class=\"line\">spark.sql(<span class=\"string\">"SELECT name, hide(ssn) As ssn FROM users"</span>).show()</span><br></pre></td></tr></table></figure>\n"},{"title":"MapReduce","excerpt":"介绍MapReduce的原理、工作流程、数据流和系统架构","date":"2024-09-30T19:42:33.000Z","math":true,"category_bar":true,"_content":"\n# MapReduce\n\n## 1. MapReduce介绍\n\n### 概览\n\n MapReduce是用来对数据进行批处理的计算框架，**可扩展性**和**可靠性**高，能够提供**容忍差错**。在MapReduce框架下，输入数据并不需要按照一定数据模型，因此该框架能处理无模式的数据。MapReduce框架将数据分为若干数据块，能够并行处理和运算，从每个数据块得到的运算结果经过汇总得到最终结果。\n\n传统的数据处理需要将数据迁移至处理数据的结点上，然而MapReduce则是将**数据处理程序迁移至储存数据的结点**上，这样避免了大批量数据的移动，同时也能够**节省带宽**、**减少处理大数据的时间**。\n\n### 工作流程\n\nMapReduce作业(Job)由一系列需要完成的工作组成，Hadoop将一项作业分为两个任务(Task)**map**和**reduce**，然后把MapReduce作业所需要的输入数据分为若干分片(Split)，为每一个分片创建一个map任务，对其中每一条记录执行map函数。\n\n\n\n**Map**阶段: 数据分为多个更小的分片(Split)，每个分片由**键值对**组成。用户定义的Map函数(mapper)对每个分片执行，由每一个原键值对，生成**一个或者多个新的键值对**。\n\n$$(K_1, V_1) \\rightarrow \\text{list}(K_2, V_2)$$\n\n**Combine**阶段: 对于大规模数据，在Map和Reduce两阶段之间交换有很高开销，而Combine阶段事实上就是对这一部分开销的**优化**。Combiner在Map阶段完成后，对数据**预先进行规约**，然后再交给Reduce阶段。比如我们想要计算每个键对应值之和，即可在combine阶段把每个键值和算出来，再传入后续的Reduce阶段，以此减少Map和Reduce之间数据交换的开销。\n\n$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_2, V_2)$$\n\n**Partition**阶段: 如果MapReduce作业涉及多个reducer，需要使用Partition函数(partitioner)，按照一定规则将mapper或combiner的输出结果分块。分块完成后，所有具有**相同键**的键值对均**位于同一分块中**。分块可以是range-based的，比如将a-m开头的键划分成分块一，将n-z开头的键划分成分块二，这样分块一给第一个reducer，分块二给第二个reducer。\n\n**Shuffle-Sort**阶段: 由partitioner产生的结果通过网络传输至执行reduce任务的结点，在每个结点中，键值对**根据键进行排序和**分组。MapReduce确保每个reducer的输入都按照键排序。在Map端，每个任务都有默认大小为100Mb的环形缓冲区，任务输出至该环形缓冲区，一旦缓冲内容达到阈值，就将内容溢出写至磁盘。在Reduce端，当一个Map任务执行完成，reduce任务就开始复制map的输出结果至磁盘或者内存(小数据)，复制完成后将map的结果合并排序。\n\n$$\\text{list}(K_2, V_2) \\rightarrow (K_2, \\text{list}(V_2))$$\n\n\n\n**Reduce**阶段: Reduce函数(reducer)对每个键的值进行规约，汇总得到最终结果。整个作业的输出结果由若干文件组成，每个文件由一个reducer得到。\n\n$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_3, V_3)$$\n\n\n\n\n## 2. MapReduce数据流\n\n### Map任务选址\n\nMap任务执行前，将优先选择HDFS中**存储输入数据的结点**，在该结点上运行Map任务，节省集群中结点之间相互传输的带宽，这一特点称为**数据本地化**。如果该结点不可用，调度程序会选择一个与该结点处于**同一机架**(rack)，且不执行Map任务的空闲结点。如果该机架中无可用结点，才从其他机架中选取结点，该操作需要机架之间的网络传输。\n\n\n\n\n对于Reduce任务，每个**reducer的输入**需要从**所有mapper的输出**得到(比如要从mapper输出结果中获取某个分块)，因此结点之间的网络传输不可避免，从而没有数据本地化的特性。\n\n### 数据分片大小\n\nMap任务对输入数据的多个分片并行执行，因此当每个分片数据比较小时，数据处理的负载平衡性会更好。但如果分片太小，每个**分片的管理和Map任务的创建开销**将占总时间消耗的主导部分。通常每个分片与一个**HDFS块**的大小一致，默认设置为**128Mb**。\n\n### Mapper输出\n\nMap任务将输出写入至**本地磁盘**，而不是HDFS，因为数据存储至HDFS会**占用集群的资源**。Map的输出是中间结果，交给Reduec任务来得到最终结果，一旦作业完成，该Map输出即可丢弃。当执行Map任务的结点发生故障后，中间结果就会丢失，需要选择另一个结点**重新执行**Map任务。\n\n\n## 3. MapReduce工作机制\n\n### YARN机制\n\nYARN是Hadoop集群资源管理系统，MapReduce等分布式计算框架作为应用层，运行在集群计算层(YARN)和存储层(HDFS)上。\n\n\n\nYARN提供两类长期运行的**守护进程**：资源管理器和结点管理器。**资源管理器**在集群中只有一个，用于**管理集群内部的资源**使用。**结点管理器**位于每个结点上，能够**启动和监控容器**，从而为应用进程的执行提供特定的运行环境和资源。\n\nYARN按照如下步骤**运行应用**：\n- 1.客户端向资源管理器请求运行一个应用主进程(application master process)\n- 2.资源管理器寻找一个结点管理器，其结点能够创建特定容器、启动应用主进程\n- 3.应用主进程向资源管理器请求更多容器\n- 4.请求得到多个容器后，应用主进程执行分布式运算\n\n\n\nYARN有三种调度方式，分别是FIFO调度、容量调度和公平调度。FIFO调度按照应用的提交顺序执行和分配资源；容量调度另外提供一个队列，保证小作业一提交就可以启动；公平调度能够在所有运行的作业之间动态地平衡资源。\n\n\n### 作业运行\n\nMapReduce作业的运行涉及如下五个实体：\n- 客户端：向YARN资源管理器提交MapReduce作业\n- YARN资源管理器：协调集群上的计算资源分配\n- YARN结点管理器：启动和监控集群中机器上用于计算的容器\n- MapReduce应用主进程：协调运行MapReduce作业中的任务，主进程和任务在容器中运行，容器由**资源管理器分配**，由**结点管理器管理**\n- 分布式文件系统HDFS：在实体之间共享作业所需文件\n\n\n\nMapReduce的作业和作业中的任务都有一个**状态**，状态包括如下信息：作业或任务的**状态**(运行中、成功完成、失败等)、Map和Reduce任务的**进度**、作业**计数器**的值、状态**消息**或描述。\n\n\n### 故障处理\n\n- 任务失败：应用主进程对任务**重新调度**，选择另一个可用的结点管理器。如果失败太多次，将不再重试，整个作业失败。\n > 可能由mapper、reducer、JVM的bug或者任务时间超时引起。\n\n- 应用主进程失败：应用主进程会向资源管理器周期性发送**心跳信息**(heartbeat)，在其失败时资源管理器能够检测到，并且由结点管理器在新的容器中启动一个新的实例。如果一个应用主进程失败了太多次，将不再重启，整个作业失败。\n > 对于应用主进程，其状态可根据**作业历史**恢复，因此**不需要重新启动**。\n\n- 结点管理器失败：该情况下，资源管理器会告知停止发送**心跳信息**的结点管理器，接着将其从可用(可启动容器)**结点池中移除**。\n > 在该故障的结点上，已完成的map任务需要重新启动，因为map阶段输出存储至该结点本地，由于失败无法访问。已完成的reduce任务无需重新执行，因为输出已储存至HDFS。\n\n- 资源管理器失败：该故障是最严重的，默认配置下资源管理器是**单点故障**，因为一旦故障，所有运行的作业都将失败且无法恢复。为获得**高可用性**，可自定义配置**运行一对资源管理器**，另一个作为备用(standby)。\n > **运行中的应用**信息可**储存在一个状态储存区**中，一般借助HDFS进行备份，备用机可根据该信息恢复资源管理器的状态。结点管理器的信息**不需要**储存，因为结点能够由资源管理器快速重启。任务的信息同样**不需要**存储，任务由应用主进程管理，不属于资源管理器状态。\n\n\n### 推测执行\n\nMapReduce作业的运行时间与执行缓慢的“拖后腿”任务相关，对于这种问题，Hadoop会在任务执行时检测其是否比预期慢，并且**启动另一个相同的任务作为备份**，这样的执行方式称为**推测执行**。调度器在执行过程中会检测同类型任务的进度，当一小部分任务的执行速度远低于平均时，将启动这一部分任务的推测副本，而当一个任务完成时，其所有副本将终止运行。","source":"_posts/MapReduce.md","raw":"---\ntitle: MapReduce\nexcerpt: 介绍MapReduce的原理、工作流程、数据流和系统架构\ndate: 2024-09-30 15:42:33\ntags:\nmath: true\ncategory_bar: true\ncategories: Realtime and Big Data Analytics\n---\n\n# MapReduce\n\n## 1. MapReduce介绍\n\n### 概览\n\n MapReduce是用来对数据进行批处理的计算框架，**可扩展性**和**可靠性**高，能够提供**容忍差错**。在MapReduce框架下，输入数据并不需要按照一定数据模型，因此该框架能处理无模式的数据。MapReduce框架将数据分为若干数据块，能够并行处理和运算，从每个数据块得到的运算结果经过汇总得到最终结果。\n\n传统的数据处理需要将数据迁移至处理数据的结点上，然而MapReduce则是将**数据处理程序迁移至储存数据的结点**上，这样避免了大批量数据的移动，同时也能够**节省带宽**、**减少处理大数据的时间**。\n\n### 工作流程\n\nMapReduce作业(Job)由一系列需要完成的工作组成，Hadoop将一项作业分为两个任务(Task)**map**和**reduce**，然后把MapReduce作业所需要的输入数据分为若干分片(Split)，为每一个分片创建一个map任务，对其中每一条记录执行map函数。\n\n\n\n**Map**阶段: 数据分为多个更小的分片(Split)，每个分片由**键值对**组成。用户定义的Map函数(mapper)对每个分片执行，由每一个原键值对，生成**一个或者多个新的键值对**。\n\n$$(K_1, V_1) \\rightarrow \\text{list}(K_2, V_2)$$\n\n**Combine**阶段: 对于大规模数据，在Map和Reduce两阶段之间交换有很高开销，而Combine阶段事实上就是对这一部分开销的**优化**。Combiner在Map阶段完成后，对数据**预先进行规约**，然后再交给Reduce阶段。比如我们想要计算每个键对应值之和，即可在combine阶段把每个键值和算出来，再传入后续的Reduce阶段，以此减少Map和Reduce之间数据交换的开销。\n\n$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_2, V_2)$$\n\n**Partition**阶段: 如果MapReduce作业涉及多个reducer，需要使用Partition函数(partitioner)，按照一定规则将mapper或combiner的输出结果分块。分块完成后，所有具有**相同键**的键值对均**位于同一分块中**。分块可以是range-based的，比如将a-m开头的键划分成分块一，将n-z开头的键划分成分块二，这样分块一给第一个reducer，分块二给第二个reducer。\n\n**Shuffle-Sort**阶段: 由partitioner产生的结果通过网络传输至执行reduce任务的结点，在每个结点中，键值对**根据键进行排序和**分组。MapReduce确保每个reducer的输入都按照键排序。在Map端，每个任务都有默认大小为100Mb的环形缓冲区，任务输出至该环形缓冲区，一旦缓冲内容达到阈值，就将内容溢出写至磁盘。在Reduce端，当一个Map任务执行完成，reduce任务就开始复制map的输出结果至磁盘或者内存(小数据)，复制完成后将map的结果合并排序。\n\n$$\\text{list}(K_2, V_2) \\rightarrow (K_2, \\text{list}(V_2))$$\n\n\n\n**Reduce**阶段: Reduce函数(reducer)对每个键的值进行规约，汇总得到最终结果。整个作业的输出结果由若干文件组成，每个文件由一个reducer得到。\n\n$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_3, V_3)$$\n\n\n\n\n## 2. MapReduce数据流\n\n### Map任务选址\n\nMap任务执行前，将优先选择HDFS中**存储输入数据的结点**，在该结点上运行Map任务，节省集群中结点之间相互传输的带宽，这一特点称为**数据本地化**。如果该结点不可用，调度程序会选择一个与该结点处于**同一机架**(rack)，且不执行Map任务的空闲结点。如果该机架中无可用结点，才从其他机架中选取结点，该操作需要机架之间的网络传输。\n\n\n\n\n对于Reduce任务，每个**reducer的输入**需要从**所有mapper的输出**得到(比如要从mapper输出结果中获取某个分块)，因此结点之间的网络传输不可避免，从而没有数据本地化的特性。\n\n### 数据分片大小\n\nMap任务对输入数据的多个分片并行执行，因此当每个分片数据比较小时，数据处理的负载平衡性会更好。但如果分片太小，每个**分片的管理和Map任务的创建开销**将占总时间消耗的主导部分。通常每个分片与一个**HDFS块**的大小一致，默认设置为**128Mb**。\n\n### Mapper输出\n\nMap任务将输出写入至**本地磁盘**，而不是HDFS，因为数据存储至HDFS会**占用集群的资源**。Map的输出是中间结果，交给Reduec任务来得到最终结果，一旦作业完成，该Map输出即可丢弃。当执行Map任务的结点发生故障后，中间结果就会丢失，需要选择另一个结点**重新执行**Map任务。\n\n\n## 3. MapReduce工作机制\n\n### YARN机制\n\nYARN是Hadoop集群资源管理系统，MapReduce等分布式计算框架作为应用层，运行在集群计算层(YARN)和存储层(HDFS)上。\n\n\n\nYARN提供两类长期运行的**守护进程**：资源管理器和结点管理器。**资源管理器**在集群中只有一个，用于**管理集群内部的资源**使用。**结点管理器**位于每个结点上，能够**启动和监控容器**，从而为应用进程的执行提供特定的运行环境和资源。\n\nYARN按照如下步骤**运行应用**：\n- 1.客户端向资源管理器请求运行一个应用主进程(application master process)\n- 2.资源管理器寻找一个结点管理器，其结点能够创建特定容器、启动应用主进程\n- 3.应用主进程向资源管理器请求更多容器\n- 4.请求得到多个容器后，应用主进程执行分布式运算\n\n\n\nYARN有三种调度方式，分别是FIFO调度、容量调度和公平调度。FIFO调度按照应用的提交顺序执行和分配资源；容量调度另外提供一个队列，保证小作业一提交就可以启动；公平调度能够在所有运行的作业之间动态地平衡资源。\n\n\n### 作业运行\n\nMapReduce作业的运行涉及如下五个实体：\n- 客户端：向YARN资源管理器提交MapReduce作业\n- YARN资源管理器：协调集群上的计算资源分配\n- YARN结点管理器：启动和监控集群中机器上用于计算的容器\n- MapReduce应用主进程：协调运行MapReduce作业中的任务，主进程和任务在容器中运行，容器由**资源管理器分配**，由**结点管理器管理**\n- 分布式文件系统HDFS：在实体之间共享作业所需文件\n\n\n\nMapReduce的作业和作业中的任务都有一个**状态**，状态包括如下信息：作业或任务的**状态**(运行中、成功完成、失败等)、Map和Reduce任务的**进度**、作业**计数器**的值、状态**消息**或描述。\n\n\n### 故障处理\n\n- 任务失败：应用主进程对任务**重新调度**，选择另一个可用的结点管理器。如果失败太多次，将不再重试，整个作业失败。\n > 可能由mapper、reducer、JVM的bug或者任务时间超时引起。\n\n- 应用主进程失败：应用主进程会向资源管理器周期性发送**心跳信息**(heartbeat)，在其失败时资源管理器能够检测到，并且由结点管理器在新的容器中启动一个新的实例。如果一个应用主进程失败了太多次，将不再重启，整个作业失败。\n > 对于应用主进程，其状态可根据**作业历史**恢复，因此**不需要重新启动**。\n\n- 结点管理器失败：该情况下，资源管理器会告知停止发送**心跳信息**的结点管理器，接着将其从可用(可启动容器)**结点池中移除**。\n > 在该故障的结点上，已完成的map任务需要重新启动，因为map阶段输出存储至该结点本地，由于失败无法访问。已完成的reduce任务无需重新执行，因为输出已储存至HDFS。\n\n- 资源管理器失败：该故障是最严重的，默认配置下资源管理器是**单点故障**，因为一旦故障，所有运行的作业都将失败且无法恢复。为获得**高可用性**，可自定义配置**运行一对资源管理器**，另一个作为备用(standby)。\n > **运行中的应用**信息可**储存在一个状态储存区**中，一般借助HDFS进行备份，备用机可根据该信息恢复资源管理器的状态。结点管理器的信息**不需要**储存，因为结点能够由资源管理器快速重启。任务的信息同样**不需要**存储，任务由应用主进程管理，不属于资源管理器状态。\n\n\n### 推测执行\n\nMapReduce作业的运行时间与执行缓慢的“拖后腿”任务相关，对于这种问题，Hadoop会在任务执行时检测其是否比预期慢，并且**启动另一个相同的任务作为备份**，这样的执行方式称为**推测执行**。调度器在执行过程中会检测同类型任务的进度，当一小部分任务的执行速度远低于平均时，将启动这一部分任务的推测副本，而当一个任务完成时，其所有副本将终止运行。","slug":"MapReduce","published":1,"updated":"2024-10-03T20:43:39.500Z","_id":"cm1pf9chy0000zaiud12mab1x","comments":1,"layout":"post","photos":[],"content":"<h1 id=\"MapReduce\"><a href=\"#MapReduce\" class=\"headerlink\" title=\"MapReduce\"></a>MapReduce</h1><h2 id=\"1-MapReduce介绍\"><a href=\"#1-MapReduce介绍\" class=\"headerlink\" title=\"1. MapReduce介绍\"></a>1. MapReduce介绍</h2><h3 id=\"概览\"><a href=\"#概览\" class=\"headerlink\" title=\"概览\"></a>概览</h3><p> MapReduce是用来对数据进行批处理的计算框架，<strong>可扩展性</strong>和<strong>可靠性</strong>高，能够提供<strong>容忍差错</strong>。在MapReduce框架下，输入数据并不需要按照一定数据模型，因此该框架能处理无模式的数据。MapReduce框架将数据分为若干数据块，能够并行处理和运算，从每个数据块得到的运算结果经过汇总得到最终结果。</p>\n<p>传统的数据处理需要将数据迁移至处理数据的结点上，然而MapReduce则是将<strong>数据处理程序迁移至储存数据的结点</strong>上，这样避免了大批量数据的移动，同时也能够<strong>节省带宽</strong>、<strong>减少处理大数据的时间</strong>。</p>\n<h3 id=\"工作流程\"><a href=\"#工作流程\" class=\"headerlink\" title=\"工作流程\"></a>工作流程</h3><p>MapReduce作业(Job)由一系列需要完成的工作组成，Hadoop将一项作业分为两个任务(Task)<strong>map</strong>和<strong>reduce</strong>，然后把MapReduce作业所需要的输入数据分为若干分片(Split)，为每一个分片创建一个map任务，对其中每一条记录执行map函数。</p>\n<p><img src=\"/../img/MapReduce/mapreduce_workflow.png\" alt=\"MapReduce的工作流程\"></p>\n<p><strong>Map</strong>阶段: 数据分为多个更小的分片(Split)，每个分片由<strong>键值对</strong>组成。用户定义的Map函数(mapper)对每个分片执行，由每一个原键值对，生成<strong>一个或者多个新的键值对</strong>。</p>\n<p>$$(K_1, V_1) \\rightarrow \\text{list}(K_2, V_2)$$</p>\n<p><strong>Combine</strong>阶段: 对于大规模数据，在Map和Reduce两阶段之间交换有很高开销，而Combine阶段事实上就是对这一部分开销的<strong>优化</strong>。Combiner在Map阶段完成后，对数据<strong>预先进行规约</strong>，然后再交给Reduce阶段。比如我们想要计算每个键对应值之和，即可在combine阶段把每个键值和算出来，再传入后续的Reduce阶段，以此减少Map和Reduce之间数据交换的开销。</p>\n<p>$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_2, V_2)$$</p>\n<p><strong>Partition</strong>阶段: 如果MapReduce作业涉及多个reducer，需要使用Partition函数(partitioner)，按照一定规则将mapper或combiner的输出结果分块。分块完成后，所有具有<strong>相同键</strong>的键值对均<strong>位于同一分块中</strong>。分块可以是range-based的，比如将a-m开头的键划分成分块一，将n-z开头的键划分成分块二，这样分块一给第一个reducer，分块二给第二个reducer。</p>\n<p><strong>Shuffle-Sort</strong>阶段: 由partitioner产生的结果通过网络传输至执行reduce任务的结点，在每个结点中，键值对<strong>根据键进行排序和</strong>分组。MapReduce确保每个reducer的输入都按照键排序。在Map端，每个任务都有默认大小为100Mb的环形缓冲区，任务输出至该环形缓冲区，一旦缓冲内容达到阈值，就将内容溢出写至磁盘。在Reduce端，当一个Map任务执行完成，reduce任务就开始复制map的输出结果至磁盘或者内存(小数据)，复制完成后将map的结果合并排序。</p>\n<p>$$\\text{list}(K_2, V_2) \\rightarrow (K_2, \\text{list}(V_2))$$</p>\n<p><img src=\"/../img/MapReduce/shufflesort.png\" alt=\"MapReduce的Shuffle和排序\"></p>\n<p><strong>Reduce</strong>阶段: Reduce函数(reducer)对每个键的值进行规约，汇总得到最终结果。整个作业的输出结果由若干文件组成，每个文件由一个reducer得到。</p>\n<p>$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_3, V_3)$$</p>\n<p><img src=\"/../img/MapReduce/mapreduce_example.png\" alt=\"MapReduce示例\"></p>\n<h2 id=\"2-MapReduce数据流\"><a href=\"#2-MapReduce数据流\" class=\"headerlink\" title=\"2. MapReduce数据流\"></a>2. MapReduce数据流</h2><h3 id=\"Map任务选址\"><a href=\"#Map任务选址\" class=\"headerlink\" title=\"Map任务选址\"></a>Map任务选址</h3><p>Map任务执行前，将优先选择HDFS中<strong>存储输入数据的结点</strong>，在该结点上运行Map任务，节省集群中结点之间相互传输的带宽，这一特点称为<strong>数据本地化</strong>。如果该结点不可用，调度程序会选择一个与该结点处于<strong>同一机架</strong>(rack)，且不执行Map任务的空闲结点。如果该机架中无可用结点，才从其他机架中选取结点，该操作需要机架之间的网络传输。</p>\n<p><img src=\"/../img/MapReduce/datalocality.png\" alt=\"Map任务选址(a为同结点，b为同机架，c为跨机架)\"></p>\n<p>对于Reduce任务，每个<strong>reducer的输入</strong>需要从<strong>所有mapper的输出</strong>得到(比如要从mapper输出结果中获取某个分块)，因此结点之间的网络传输不可避免，从而没有数据本地化的特性。</p>\n<h3 id=\"数据分片大小\"><a href=\"#数据分片大小\" class=\"headerlink\" title=\"数据分片大小\"></a>数据分片大小</h3><p>Map任务对输入数据的多个分片并行执行，因此当每个分片数据比较小时，数据处理的负载平衡性会更好。但如果分片太小，每个<strong>分片的管理和Map任务的创建开销</strong>将占总时间消耗的主导部分。通常每个分片与一个<strong>HDFS块</strong>的大小一致，默认设置为<strong>128Mb</strong>。</p>\n<h3 id=\"Mapper输出\"><a href=\"#Mapper输出\" class=\"headerlink\" title=\"Mapper输出\"></a>Mapper输出</h3><p>Map任务将输出写入至<strong>本地磁盘</strong>，而不是HDFS，因为数据存储至HDFS会<strong>占用集群的资源</strong>。Map的输出是中间结果，交给Reduec任务来得到最终结果，一旦作业完成，该Map输出即可丢弃。当执行Map任务的结点发生故障后，中间结果就会丢失，需要选择另一个结点<strong>重新执行</strong>Map任务。</p>\n<h2 id=\"3-MapReduce工作机制\"><a href=\"#3-MapReduce工作机制\" class=\"headerlink\" title=\"3. MapReduce工作机制\"></a>3. MapReduce工作机制</h2><h3 id=\"YARN机制\"><a href=\"#YARN机制\" class=\"headerlink\" title=\"YARN机制\"></a>YARN机制</h3><p>YARN是Hadoop集群资源管理系统，MapReduce等分布式计算框架作为应用层，运行在集群计算层(YARN)和存储层(HDFS)上。</p>\n<p><img src=\"/../img/MapReduce/yarnlayer.png\" alt=\"YARN与其他层级的关系\"></p>\n<p>YARN提供两类长期运行的<strong>守护进程</strong>：资源管理器和结点管理器。<strong>资源管理器</strong>在集群中只有一个，用于<strong>管理集群内部的资源</strong>使用。<strong>结点管理器</strong>位于每个结点上，能够<strong>启动和监控容器</strong>，从而为应用进程的执行提供特定的运行环境和资源。</p>\n<p>YARN按照如下步骤<strong>运行应用</strong>：</p>\n<ul>\n<li>1.客户端向资源管理器请求运行一个应用主进程(application master process)</li>\n<li>2.资源管理器寻找一个结点管理器，其结点能够创建特定容器、启动应用主进程</li>\n<li>3.应用主进程向资源管理器请求更多容器</li>\n<li>4.请求得到多个容器后，应用主进程执行分布式运算</li>\n</ul>\n<p><img src=\"/../img/MapReduce/yarnapp.png\" alt=\"YARN应用的运行机制\"></p>\n<p>YARN有三种调度方式，分别是FIFO调度、容量调度和公平调度。FIFO调度按照应用的提交顺序执行和分配资源；容量调度另外提供一个队列，保证小作业一提交就可以启动；公平调度能够在所有运行的作业之间动态地平衡资源。</p>\n<h3 id=\"作业运行\"><a href=\"#作业运行\" class=\"headerlink\" title=\"作业运行\"></a>作业运行</h3><p>MapReduce作业的运行涉及如下五个实体：</p>\n<ul>\n<li>客户端：向YARN资源管理器提交MapReduce作业</li>\n<li>YARN资源管理器：协调集群上的计算资源分配</li>\n<li>YARN结点管理器：启动和监控集群中机器上用于计算的容器</li>\n<li>MapReduce应用主进程：协调运行MapReduce作业中的任务，主进程和任务在容器中运行，容器由<strong>资源管理器分配</strong>，由<strong>结点管理器管理</strong></li>\n<li>分布式文件系统HDFS：在实体之间共享作业所需文件</li>\n</ul>\n<p><img src=\"/../img/MapReduce/mapreduce_jobexec.png\" alt=\"Hadoop的MapReduce作业运行机制\"></p>\n<p>MapReduce的作业和作业中的任务都有一个<strong>状态</strong>，状态包括如下信息：作业或任务的<strong>状态</strong>(运行中、成功完成、失败等)、Map和Reduce任务的<strong>进度</strong>、作业<strong>计数器</strong>的值、状态<strong>消息</strong>或描述。</p>\n<h3 id=\"故障处理\"><a href=\"#故障处理\" class=\"headerlink\" title=\"故障处理\"></a>故障处理</h3><ul>\n<li><p>任务失败：应用主进程对任务<strong>重新调度</strong>，选择另一个可用的结点管理器。如果失败太多次，将不再重试，整个作业失败。</p>\n<blockquote>\n<p>可能由mapper、reducer、JVM的bug或者任务时间超时引起。</p>\n</blockquote>\n</li>\n<li><p>应用主进程失败：应用主进程会向资源管理器周期性发送<strong>心跳信息</strong>(heartbeat)，在其失败时资源管理器能够检测到，并且由结点管理器在新的容器中启动一个新的实例。如果一个应用主进程失败了太多次，将不再重启，整个作业失败。</p>\n<blockquote>\n<p>对于应用主进程，其状态可根据<strong>作业历史</strong>恢复，因此<strong>不需要重新启动</strong>。</p>\n</blockquote>\n</li>\n<li><p>结点管理器失败：该情况下，资源管理器会告知停止发送<strong>心跳信息</strong>的结点管理器，接着将其从可用(可启动容器)<strong>结点池中移除</strong>。</p>\n<blockquote>\n<p>在该故障的结点上，已完成的map任务需要重新启动，因为map阶段输出存储至该结点本地，由于失败无法访问。已完成的reduce任务无需重新执行，因为输出已储存至HDFS。</p>\n</blockquote>\n</li>\n<li><p>资源管理器失败：该故障是最严重的，默认配置下资源管理器是<strong>单点故障</strong>，因为一旦故障，所有运行的作业都将失败且无法恢复。为获得<strong>高可用性</strong>，可自定义配置<strong>运行一对资源管理器</strong>，另一个作为备用(standby)。</p>\n<blockquote>\n<p><strong>运行中的应用</strong>信息可<strong>储存在一个状态储存区</strong>中，一般借助HDFS进行备份，备用机可根据该信息恢复资源管理器的状态。结点管理器的信息<strong>不需要</strong>储存，因为结点能够由资源管理器快速重启。任务的信息同样<strong>不需要</strong>存储，任务由应用主进程管理，不属于资源管理器状态。</p>\n</blockquote>\n</li>\n</ul>\n<h3 id=\"推测执行\"><a href=\"#推测执行\" class=\"headerlink\" title=\"推测执行\"></a>推测执行</h3><p>MapReduce作业的运行时间与执行缓慢的“拖后腿”任务相关，对于这种问题，Hadoop会在任务执行时检测其是否比预期慢，并且<strong>启动另一个相同的任务作为备份</strong>，这样的执行方式称为<strong>推测执行</strong>。调度器在执行过程中会检测同类型任务的进度，当一小部分任务的执行速度远低于平均时，将启动这一部分任务的推测副本，而当一个任务完成时，其所有副本将终止运行。</p>\n","more":"<h1 id=\"MapReduce\"><a href=\"#MapReduce\" class=\"headerlink\" title=\"MapReduce\"></a>MapReduce</h1><h2 id=\"1-MapReduce介绍\"><a href=\"#1-MapReduce介绍\" class=\"headerlink\" title=\"1. MapReduce介绍\"></a>1. MapReduce介绍</h2><h3 id=\"概览\"><a href=\"#概览\" class=\"headerlink\" title=\"概览\"></a>概览</h3><p> MapReduce是用来对数据进行批处理的计算框架，<strong>可扩展性</strong>和<strong>可靠性</strong>高，能够提供<strong>容忍差错</strong>。在MapReduce框架下，输入数据并不需要按照一定数据模型，因此该框架能处理无模式的数据。MapReduce框架将数据分为若干数据块，能够并行处理和运算，从每个数据块得到的运算结果经过汇总得到最终结果。</p>\n<p>传统的数据处理需要将数据迁移至处理数据的结点上，然而MapReduce则是将<strong>数据处理程序迁移至储存数据的结点</strong>上，这样避免了大批量数据的移动，同时也能够<strong>节省带宽</strong>、<strong>减少处理大数据的时间</strong>。</p>\n<h3 id=\"工作流程\"><a href=\"#工作流程\" class=\"headerlink\" title=\"工作流程\"></a>工作流程</h3><p>MapReduce作业(Job)由一系列需要完成的工作组成，Hadoop将一项作业分为两个任务(Task)<strong>map</strong>和<strong>reduce</strong>，然后把MapReduce作业所需要的输入数据分为若干分片(Split)，为每一个分片创建一个map任务，对其中每一条记录执行map函数。</p>\n<p><img src=\"/../img/MapReduce/mapreduce_workflow.png\" alt=\"MapReduce的工作流程\"></p>\n<p><strong>Map</strong>阶段: 数据分为多个更小的分片(Split)，每个分片由<strong>键值对</strong>组成。用户定义的Map函数(mapper)对每个分片执行，由每一个原键值对，生成<strong>一个或者多个新的键值对</strong>。</p>\n<p>$$(K_1, V_1) \\rightarrow \\text{list}(K_2, V_2)$$</p>\n<p><strong>Combine</strong>阶段: 对于大规模数据，在Map和Reduce两阶段之间交换有很高开销，而Combine阶段事实上就是对这一部分开销的<strong>优化</strong>。Combiner在Map阶段完成后，对数据<strong>预先进行规约</strong>，然后再交给Reduce阶段。比如我们想要计算每个键对应值之和，即可在combine阶段把每个键值和算出来，再传入后续的Reduce阶段，以此减少Map和Reduce之间数据交换的开销。</p>\n<p>$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_2, V_2)$$</p>\n<p><strong>Partition</strong>阶段: 如果MapReduce作业涉及多个reducer，需要使用Partition函数(partitioner)，按照一定规则将mapper或combiner的输出结果分块。分块完成后，所有具有<strong>相同键</strong>的键值对均<strong>位于同一分块中</strong>。分块可以是range-based的，比如将a-m开头的键划分成分块一，将n-z开头的键划分成分块二，这样分块一给第一个reducer，分块二给第二个reducer。</p>\n<p><strong>Shuffle-Sort</strong>阶段: 由partitioner产生的结果通过网络传输至执行reduce任务的结点，在每个结点中，键值对<strong>根据键进行排序和</strong>分组。MapReduce确保每个reducer的输入都按照键排序。在Map端，每个任务都有默认大小为100Mb的环形缓冲区，任务输出至该环形缓冲区，一旦缓冲内容达到阈值，就将内容溢出写至磁盘。在Reduce端，当一个Map任务执行完成，reduce任务就开始复制map的输出结果至磁盘或者内存(小数据)，复制完成后将map的结果合并排序。</p>\n<p>$$\\text{list}(K_2, V_2) \\rightarrow (K_2, \\text{list}(V_2))$$</p>\n<p><img src=\"/../img/MapReduce/shufflesort.png\" alt=\"MapReduce的Shuffle和排序\"></p>\n<p><strong>Reduce</strong>阶段: Reduce函数(reducer)对每个键的值进行规约，汇总得到最终结果。整个作业的输出结果由若干文件组成，每个文件由一个reducer得到。</p>\n<p>$$(K_2, \\text{list}(V_2)) \\rightarrow \\text{list}(K_3, V_3)$$</p>\n<p><img src=\"/../img/MapReduce/mapreduce_example.png\" alt=\"MapReduce示例\"></p>\n<h2 id=\"2-MapReduce数据流\"><a href=\"#2-MapReduce数据流\" class=\"headerlink\" title=\"2. MapReduce数据流\"></a>2. MapReduce数据流</h2><h3 id=\"Map任务选址\"><a href=\"#Map任务选址\" class=\"headerlink\" title=\"Map任务选址\"></a>Map任务选址</h3><p>Map任务执行前，将优先选择HDFS中<strong>存储输入数据的结点</strong>，在该结点上运行Map任务，节省集群中结点之间相互传输的带宽，这一特点称为<strong>数据本地化</strong>。如果该结点不可用，调度程序会选择一个与该结点处于<strong>同一机架</strong>(rack)，且不执行Map任务的空闲结点。如果该机架中无可用结点，才从其他机架中选取结点，该操作需要机架之间的网络传输。</p>\n<p><img src=\"/../img/MapReduce/datalocality.png\" alt=\"Map任务选址(a为同结点，b为同机架，c为跨机架)\"></p>\n<p>对于Reduce任务，每个<strong>reducer的输入</strong>需要从<strong>所有mapper的输出</strong>得到(比如要从mapper输出结果中获取某个分块)，因此结点之间的网络传输不可避免，从而没有数据本地化的特性。</p>\n<h3 id=\"数据分片大小\"><a href=\"#数据分片大小\" class=\"headerlink\" title=\"数据分片大小\"></a>数据分片大小</h3><p>Map任务对输入数据的多个分片并行执行，因此当每个分片数据比较小时，数据处理的负载平衡性会更好。但如果分片太小，每个<strong>分片的管理和Map任务的创建开销</strong>将占总时间消耗的主导部分。通常每个分片与一个<strong>HDFS块</strong>的大小一致，默认设置为<strong>128Mb</strong>。</p>\n<h3 id=\"Mapper输出\"><a href=\"#Mapper输出\" class=\"headerlink\" title=\"Mapper输出\"></a>Mapper输出</h3><p>Map任务将输出写入至<strong>本地磁盘</strong>，而不是HDFS，因为数据存储至HDFS会<strong>占用集群的资源</strong>。Map的输出是中间结果，交给Reduec任务来得到最终结果，一旦作业完成，该Map输出即可丢弃。当执行Map任务的结点发生故障后，中间结果就会丢失，需要选择另一个结点<strong>重新执行</strong>Map任务。</p>\n<h2 id=\"3-MapReduce工作机制\"><a href=\"#3-MapReduce工作机制\" class=\"headerlink\" title=\"3. MapReduce工作机制\"></a>3. MapReduce工作机制</h2><h3 id=\"YARN机制\"><a href=\"#YARN机制\" class=\"headerlink\" title=\"YARN机制\"></a>YARN机制</h3><p>YARN是Hadoop集群资源管理系统，MapReduce等分布式计算框架作为应用层，运行在集群计算层(YARN)和存储层(HDFS)上。</p>\n<p><img src=\"/../img/MapReduce/yarnlayer.png\" alt=\"YARN与其他层级的关系\"></p>\n<p>YARN提供两类长期运行的<strong>守护进程</strong>：资源管理器和结点管理器。<strong>资源管理器</strong>在集群中只有一个，用于<strong>管理集群内部的资源</strong>使用。<strong>结点管理器</strong>位于每个结点上，能够<strong>启动和监控容器</strong>，从而为应用进程的执行提供特定的运行环境和资源。</p>\n<p>YARN按照如下步骤<strong>运行应用</strong>：</p>\n<ul>\n<li>1.客户端向资源管理器请求运行一个应用主进程(application master process)</li>\n<li>2.资源管理器寻找一个结点管理器，其结点能够创建特定容器、启动应用主进程</li>\n<li>3.应用主进程向资源管理器请求更多容器</li>\n<li>4.请求得到多个容器后，应用主进程执行分布式运算</li>\n</ul>\n<p><img src=\"/../img/MapReduce/yarnapp.png\" alt=\"YARN应用的运行机制\"></p>\n<p>YARN有三种调度方式，分别是FIFO调度、容量调度和公平调度。FIFO调度按照应用的提交顺序执行和分配资源；容量调度另外提供一个队列，保证小作业一提交就可以启动；公平调度能够在所有运行的作业之间动态地平衡资源。</p>\n<h3 id=\"作业运行\"><a href=\"#作业运行\" class=\"headerlink\" title=\"作业运行\"></a>作业运行</h3><p>MapReduce作业的运行涉及如下五个实体：</p>\n<ul>\n<li>客户端：向YARN资源管理器提交MapReduce作业</li>\n<li>YARN资源管理器：协调集群上的计算资源分配</li>\n<li>YARN结点管理器：启动和监控集群中机器上用于计算的容器</li>\n<li>MapReduce应用主进程：协调运行MapReduce作业中的任务，主进程和任务在容器中运行，容器由<strong>资源管理器分配</strong>，由<strong>结点管理器管理</strong></li>\n<li>分布式文件系统HDFS：在实体之间共享作业所需文件</li>\n</ul>\n<p><img src=\"/../img/MapReduce/mapreduce_jobexec.png\" alt=\"Hadoop的MapReduce作业运行机制\"></p>\n<p>MapReduce的作业和作业中的任务都有一个<strong>状态</strong>，状态包括如下信息：作业或任务的<strong>状态</strong>(运行中、成功完成、失败等)、Map和Reduce任务的<strong>进度</strong>、作业<strong>计数器</strong>的值、状态<strong>消息</strong>或描述。</p>\n<h3 id=\"故障处理\"><a href=\"#故障处理\" class=\"headerlink\" title=\"故障处理\"></a>故障处理</h3><ul>\n<li><p>任务失败：应用主进程对任务<strong>重新调度</strong>，选择另一个可用的结点管理器。如果失败太多次，将不再重试，整个作业失败。</p>\n<blockquote>\n<p>可能由mapper、reducer、JVM的bug或者任务时间超时引起。</p>\n</blockquote>\n</li>\n<li><p>应用主进程失败：应用主进程会向资源管理器周期性发送<strong>心跳信息</strong>(heartbeat)，在其失败时资源管理器能够检测到，并且由结点管理器在新的容器中启动一个新的实例。如果一个应用主进程失败了太多次，将不再重启，整个作业失败。</p>\n<blockquote>\n<p>对于应用主进程，其状态可根据<strong>作业历史</strong>恢复，因此<strong>不需要重新启动</strong>。</p>\n</blockquote>\n</li>\n<li><p>结点管理器失败：该情况下，资源管理器会告知停止发送<strong>心跳信息</strong>的结点管理器，接着将其从可用(可启动容器)<strong>结点池中移除</strong>。</p>\n<blockquote>\n<p>在该故障的结点上，已完成的map任务需要重新启动，因为map阶段输出存储至该结点本地，由于失败无法访问。已完成的reduce任务无需重新执行，因为输出已储存至HDFS。</p>\n</blockquote>\n</li>\n<li><p>资源管理器失败：该故障是最严重的，默认配置下资源管理器是<strong>单点故障</strong>，因为一旦故障，所有运行的作业都将失败且无法恢复。为获得<strong>高可用性</strong>，可自定义配置<strong>运行一对资源管理器</strong>，另一个作为备用(standby)。</p>\n<blockquote>\n<p><strong>运行中的应用</strong>信息可<strong>储存在一个状态储存区</strong>中，一般借助HDFS进行备份，备用机可根据该信息恢复资源管理器的状态。结点管理器的信息<strong>不需要</strong>储存，因为结点能够由资源管理器快速重启。任务的信息同样<strong>不需要</strong>存储，任务由应用主进程管理，不属于资源管理器状态。</p>\n</blockquote>\n</li>\n</ul>\n<h3 id=\"推测执行\"><a href=\"#推测执行\" class=\"headerlink\" title=\"推测执行\"></a>推测执行</h3><p>MapReduce作业的运行时间与执行缓慢的“拖后腿”任务相关，对于这种问题，Hadoop会在任务执行时检测其是否比预期慢，并且<strong>启动另一个相同的任务作为备份</strong>，这样的执行方式称为<strong>推测执行</strong>。调度器在执行过程中会检测同类型任务的进度，当一小部分任务的执行速度远低于平均时，将启动这一部分任务的推测副本，而当一个任务完成时，其所有副本将终止运行。</p>\n"},{"title":"MapReduce Programming","excerpt":"介绍MapReduce的程序设计模式","date":"2024-10-03T20:41:02.000Z","math":true,"category_bar":true,"_content":"\n```java\npackage org.apache.hadoop.io;\n\nimport java.io.DataOutput;\nimport java.io.DataInput;\nimport java.io.IOException;\n\npublic interface Writable {\n void write(DataOutput out) throws IOException;\n void readFields(DataInput in) throws IOException;\n}\n```","source":"_posts/MapReduce-Programming.md","raw":"---\ntitle: MapReduce Programming\nexcerpt: 介绍MapReduce的程序设计模式\ndate: 2024-10-03 16:41:02\ntags:\nmath: true\ncategory_bar: true\ncategories: Realtime and Big Data Analytics\n---\n\n```java\npackage org.apache.hadoop.io;\n\nimport java.io.DataOutput;\nimport java.io.DataInput;\nimport java.io.IOException;\n\npublic interface Writable {\n void write(DataOutput out) throws IOException;\n void readFields(DataInput in) throws IOException;\n}\n```","slug":"MapReduce-Programming","published":1,"updated":"2024-10-03T20:43:34.211Z","_id":"cm1trk2w600007iiv7dtmd3zs","comments":1,"layout":"post","photos":[],"content":"<figure class=\"highlight java\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br></pre></td><td class=\"code\"><pre><code class=\"hljs java\"><span class=\"hljs-keyword\">package</span> org.apache.hadoop.io;<br><br><span class=\"hljs-keyword\">import</span> java.io.DataOutput;<br><span class=\"hljs-keyword\">import</span> java.io.DataInput;<br><span class=\"hljs-keyword\">import</span> java.io.IOException;<br><br><span class=\"hljs-keyword\">public</span> <span class=\"hljs-keyword\">interface</span> <span class=\"hljs-title class_\">Writable</span> {<br> <span class=\"hljs-keyword\">void</span> <span class=\"hljs-title function_\">write</span><span class=\"hljs-params\">(DataOutput out)</span> <span class=\"hljs-keyword\">throws</span> IOException;<br> <span class=\"hljs-keyword\">void</span> <span class=\"hljs-title function_\">readFields</span><span class=\"hljs-params\">(DataInput in)</span> <span class=\"hljs-keyword\">throws</span> IOException;<br>}<br></code></pre></td></tr></table></figure>","more":"<figure class=\"highlight java\"><table><tr><td class=\"gutter\"><pre><span class=\"line\">1</span><br><span class=\"line\">2</span><br><span class=\"line\">3</span><br><span class=\"line\">4</span><br><span class=\"line\">5</span><br><span class=\"line\">6</span><br><span class=\"line\">7</span><br><span class=\"line\">8</span><br><span class=\"line\">9</span><br><span class=\"line\">10</span><br></pre></td><td class=\"code\"><pre><code class=\"hljs java\"><span class=\"hljs-keyword\">package</span> org.apache.hadoop.io;<br><br><span class=\"hljs-keyword\">import</span> java.io.DataOutput;<br><span class=\"hljs-keyword\">import</span> java.io.DataInput;<br><span class=\"hljs-keyword\">import</span> java.io.IOException;<br><br><span class=\"hljs-keyword\">public</span> <span class=\"hljs-keyword\">interface</span> <span class=\"hljs-title class_\">Writable</span> {<br> <span class=\"hljs-keyword\">void</span> <span class=\"hljs-title function_\">write</span><span class=\"hljs-params\">(DataOutput out)</span> <span class=\"hljs-keyword\">throws</span> IOException;<br> <span class=\"hljs-keyword\">void</span> <span class=\"hljs-title function_\">readFields</span><span class=\"hljs-params\">(DataInput in)</span> <span class=\"hljs-keyword\">throws</span> IOException;<br>}<br></code></pre></td></tr></table></figure>"}],"PostAsset":[],"PostCategory":[{"post_id":"cm1pf9chy0000zaiud12mab1x","category_id":"cm1tpffqr0002h5iv3v7q9v72","_id":"cm1tpffqs0003h5iv4dh6erlu"},{"post_id":"cm1trk2w600007iiv7dtmd3zs","category_id":"cm1tpffqr0002h5iv3v7q9v72","_id":"cm1trk2wb00017iive56z8xee"}],"PostTag":[],"Tag":[]}}