{"id":1333,"date":"2026-01-12T11:01:13","date_gmt":"2026-01-12T03:01:13","guid":{"rendered":"https:\/\/www.52runoob.com\/?p=1333"},"modified":"2026-01-12T11:01:13","modified_gmt":"2026-01-12T03:01:13","slug":"%e5%a4%a7%e6%95%b0%e6%8d%aespark%ef%bc%88%e4%b8%83%e5%8d%81%e4%ba%94%ef%bc%89%ef%bc%9aaction%e8%a1%8c%e5%8a%a8%e7%ae%97%e5%ad%90foreachpartition%e5%92%8ccount%e4%bd%bf%e7%94%a8%e6%a1%88%e4%be%8b","status":"publish","type":"post","link":"https:\/\/www.52runoob.com\/index.php\/2026\/01\/12\/%e5%a4%a7%e6%95%b0%e6%8d%aespark%ef%bc%88%e4%b8%83%e5%8d%81%e4%ba%94%ef%bc%89%ef%bc%9aaction%e8%a1%8c%e5%8a%a8%e7%ae%97%e5%ad%90foreachpartition%e5%92%8ccount%e4%bd%bf%e7%94%a8%e6%a1%88%e4%be%8b\/","title":{"rendered":"\u5927\u6570\u636eSpark\uff08\u4e03\u5341\u4e94\uff09\uff1aAction\u884c\u52a8\u7b97\u5b50foreachpartition\u548ccount\u4f7f\u7528\u6848\u4f8b"},"content":{"rendered":"\n<p>\u4e0b\u9762\u662f\u4e00\u7bc7<strong>\u5b8c\u6574\u3001\u53ef\u76f4\u63a5\u53d1\u5e03\u7684\u6280\u672f\u6559\u7a0b\u6587\u7ae0<\/strong>\uff0c\u6807\u9898\u4e0e\u98ce\u683c\u7b26\u5408\u4f60\u4e4b\u524d\u7684 Spark \u7cfb\u5217\u5185\u5bb9 \ud83d\udc47<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h1 class=\"wp-block-heading\">\u5927\u6570\u636e Spark\uff08\u4e03\u5341\u4e94\uff09\uff1aAction \u884c\u52a8\u7b97\u5b50 foreachPartition \u548c count \u4f7f\u7528\u6848\u4f8b<\/h1>\n\n\n\n<p>\u5728 Spark \u4e2d\uff0c\u7b97\u5b50\u5927\u4f53\u5206\u4e3a\u4e24\u7c7b\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Transformation\uff08\u8f6c\u6362\u7b97\u5b50\uff09<\/strong>\uff1a\u8fd4\u56de\u65b0\u7684 RDD\uff0c\u5ef6\u8fdf\u6267\u884c<\/li>\n\n\n\n<li><strong>Action\uff08\u884c\u52a8\u7b97\u5b50\uff09<\/strong>\uff1a\u89e6\u53d1\u771f\u6b63\u8ba1\u7b97\uff0c\u8fd4\u56de\u7ed3\u679c\u6216\u6267\u884c\u526f\u4f5c\u7528<\/li>\n<\/ul>\n\n\n\n<p>\u672c\u7bc7\u6211\u4eec\u91cd\u70b9\u8bb2\u89e3\u4e24\u4e2a\u5e38\u7528\u4f46\u7528\u9014\u5b8c\u5168\u4e0d\u540c\u7684 <strong>Action \u884c\u52a8\u7b97\u5b50<\/strong>\uff1a<\/p>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<p><strong>foreachPartition<\/strong> \u548c <strong>count<\/strong><\/p>\n<\/blockquote>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e00\u3001count \u7b97\u5b50\u8be6\u89e3<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">1\ufe0f\u20e3 count \u662f\u4ec0\u4e48\uff1f<\/h3>\n\n\n\n<p><code>count<\/code> \u662f\u6700\u57fa\u7840\u3001\u6700\u5e38\u89c1\u7684 Action \u7b97\u5b50\u4e4b\u4e00\uff0c\u7528\u4e8e\uff1a<\/p>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<p><strong>\u7edf\u8ba1 RDD \u4e2d\u5143\u7d20\u7684\u6570\u91cf<\/strong><\/p>\n<\/blockquote>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\ndef count(): Long\n\n<\/pre><\/div>\n\n\n<p>\u5b83\u4f1a\u89e6\u53d1 Spark \u4f5c\u4e1a\u6267\u884c\uff0c\u5e76\u8fd4\u56de\u4e00\u4e2a <code>Long<\/code> \u7c7b\u578b\u7684\u7ed3\u679c\u3002<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">2\ufe0f\u20e3 \u4f7f\u7528\u793a\u4f8b<\/h3>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\nval conf = new SparkConf().setMaster(&quot;local&#x5B;*]&quot;).setAppName(&quot;CountDemo&quot;)\nval sc = new SparkContext(conf)\n\nval rdd = sc.parallelize(List(1, 2, 3, 4, 5))\n\nval result = rdd.count()\nprintln(result)\n\nsc.stop()\n\n<\/pre><\/div>\n\n\n<p><strong>\u8f93\u51fa\u7ed3\u679c\uff1a<\/strong><\/p>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\n5\n\n<\/pre><\/div>\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">3\ufe0f\u20e3 \u5e95\u5c42\u6267\u884c\u903b\u8f91\u8bf4\u660e<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><code>count<\/code> \u4f1a\u904d\u5386 <strong>\u6240\u6709\u5206\u533a<\/strong><\/li>\n\n\n\n<li>\u6bcf\u4e2a\u5206\u533a\u672c\u5730\u7edf\u8ba1\u5143\u7d20\u6570\u91cf<\/li>\n\n\n\n<li>Driver \u7aef\u6c47\u603b\u6240\u6709\u5206\u533a\u7ed3\u679c<\/li>\n<\/ul>\n\n\n\n<p>\u26a0\ufe0f \u6ce8\u610f\uff1a<br><code>count<\/code> <strong>\u4e0d\u4f1a\u8fd4\u56de\u6570\u636e\u5185\u5bb9<\/strong>\uff0c\u53ea\u8fd4\u56de\u6570\u91cf\uff0c\u56e0\u6b64\u975e\u5e38\u9002\u5408\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u7edf\u8ba1\u6570\u636e\u89c4\u6a21<\/li>\n\n\n\n<li>\u9a8c\u8bc1\u6570\u636e\u662f\u5426\u4e3a\u7a7a<\/li>\n\n\n\n<li>\u8c03\u8bd5\u7ba1\u9053\u662f\u5426\u751f\u6548<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e8c\u3001foreachPartition \u7b97\u5b50\u8be6\u89e3<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">1\ufe0f\u20e3 foreachPartition \u662f\u4ec0\u4e48\uff1f<\/h3>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\ndef foreachPartition(f: Iterator&#x5B;T] =&gt; Unit): Unit\n\n<\/pre><\/div>\n\n\n<p><code>foreachPartition<\/code> \u662f\u4e00\u4e2a <strong>\u4ee5\u5206\u533a\u4e3a\u5355\u4f4d\u6267\u884c\u7684 Action \u7b97\u5b50<\/strong>\u3002<\/p>\n\n\n\n<p>\u4e0e <code>foreach<\/code> \u7684\u6700\u5927\u533a\u522b\u5728\u4e8e\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th>\u7b97\u5b50<\/th><th>\u6267\u884c\u7c92\u5ea6<\/th><\/tr><\/thead><tbody><tr><td>foreach<\/td><td>\u6bcf\u6761\u6570\u636e<\/td><\/tr><tr><td>foreachPartition<\/td><td>\u6bcf\u4e2a\u5206\u533a<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">2\ufe0f\u20e3 foreach vs foreachPartition \u5bf9\u6bd4<\/h3>\n\n\n\n<h4 class=\"wp-block-heading\">foreach \u793a\u4f8b\uff08\u4e0d\u63a8\u8350\u7528\u4e8e\u5916\u90e8\u7cfb\u7edf\uff09<\/h4>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\nrdd.foreach(x =&gt; {\n  println(x)\n})\n\n<\/pre><\/div>\n\n\n<p>\u95ee\u9898\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u6bcf\u6761\u6570\u636e\u90fd\u4f1a\u6267\u884c\u4e00\u6b21\u903b\u8f91<\/li>\n\n\n\n<li>\u5982\u679c\u6d89\u53ca\u6570\u636e\u5e93 \/ \u7f51\u7edc IO\uff0c\u5f00\u9500\u5de8\u5927<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h4 class=\"wp-block-heading\">foreachPartition \u793a\u4f8b\uff08\u63a8\u8350\uff09<\/h4>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\nrdd.foreachPartition(iter =&gt; {\n  while (iter.hasNext) {\n    println(iter.next())\n  }\n})\n\n<\/pre><\/div>\n\n\n<p>\u4f18\u52bf\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u6bcf\u4e2a\u5206\u533a\u53ea\u6267\u884c\u4e00\u6b21<\/li>\n\n\n\n<li>\u975e\u5e38\u9002\u5408 <strong>\u6279\u91cf\u5199\u5165\u5916\u90e8\u7cfb\u7edf<\/strong><\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">3\ufe0f\u20e3 \u5178\u578b\u4f7f\u7528\u573a\u666f\uff1a\u5199\u5165\u6570\u636e\u5e93<\/h3>\n\n\n\n<h4 class=\"wp-block-heading\">\u274c \u9519\u8bef\u793a\u4f8b\uff08foreach\uff09<\/h4>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\nrdd.foreach(data =&gt; {\n  val conn = getConnection()\n  insert(data)\n  conn.close()\n})\n\n<\/pre><\/div>\n\n\n<p>\u95ee\u9898\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u6bcf\u6761\u6570\u636e\u521b\u5efa\u4e00\u6b21\u8fde\u63a5<\/li>\n\n\n\n<li>\u6781\u6613\u538b\u57ae\u6570\u636e\u5e93<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h4 class=\"wp-block-heading\">\u2705 \u6b63\u786e\u793a\u4f8b\uff08foreachPartition\uff09<\/h4>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\nrdd.foreachPartition(iter =&gt; {\n  val conn = getConnection()\n  iter.foreach(data =&gt; {\n    insert(data)\n  })\n  conn.close()\n})\n\n<\/pre><\/div>\n\n\n<p>\u2714 \u6bcf\u4e2a\u5206\u533a\u4e00\u4e2a\u8fde\u63a5<br>\u2714 \u6781\u5927\u51cf\u5c11\u8d44\u6e90\u6d88\u8017<br>\u2714 Spark \u5b98\u65b9\u63a8\u8350\u5199\u6cd5<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e09\u3001\u7efc\u5408\u6848\u4f8b\uff1acount + foreachPartition<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">\u9700\u6c42\u8bf4\u660e<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u7edf\u8ba1\u6570\u636e\u6761\u6570<\/li>\n\n\n\n<li>\u6309\u5206\u533a\u8f93\u51fa\u6570\u636e\u5185\u5bb9<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">\u793a\u4f8b\u4ee3\u7801<\/h3>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\nval conf = new SparkConf().setMaster(&quot;local&#x5B;*]&quot;).setAppName(&quot;ActionDemo&quot;)\nval sc = new SparkContext(conf)\n\nval rdd = sc.parallelize(List(&quot;a&quot;, &quot;b&quot;, &quot;c&quot;, &quot;d&quot;, &quot;e&quot;), 2)\n\n\/\/ \u7edf\u8ba1\u6570\u636e\u91cf\nval cnt = rdd.count()\nprintln(s&quot;\u6570\u636e\u603b\u91cf\uff1a$cnt&quot;)\n\n\/\/ \u6309\u5206\u533a\u5904\u7406\nrdd.foreachPartition(iter =&gt; {\n  println(&quot;====== \u5206\u533a\u5f00\u59cb ======&quot;)\n  iter.foreach(println)\n})\n\nsc.stop()\n\n<\/pre><\/div>\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">\u6267\u884c\u7ed3\u679c\u793a\u610f<\/h3>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\n\u6570\u636e\u603b\u91cf\uff1a5\n====== \u5206\u533a\u5f00\u59cb ======\na\nb\n====== \u5206\u533a\u5f00\u59cb ======\nc\nd\ne\n\n<\/pre><\/div>\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\">\u56db\u3001\u6ce8\u610f\u4e8b\u9879\u4e0e\u6700\u4f73\u5b9e\u8df5<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">\u26a0\ufe0f 1\ufe0f\u20e3 foreachPartition \u662f Action<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u4f1a\u89e6\u53d1 Spark \u4f5c\u4e1a\u6267\u884c<\/li>\n\n\n\n<li>\u4e0e count \u4e00\u6837\u5c5e\u4e8e\u6700\u7ec8\u7b97\u5b50<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">\u26a0\ufe0f 2\ufe0f\u20e3 foreachPartition \u65e0\u8fd4\u56de\u503c<\/h3>\n\n\n<div class=\"wp-block-syntaxhighlighter-code \"><pre class=\"brush: plain; title: ; notranslate\" title=\"\">\nforeachPartition(...): Unit\n\n<\/pre><\/div>\n\n\n<p>\ud83d\udc49 <strong>\u4e0d\u80fd\u7528\u4e8e\u7ed3\u679c\u8ba1\u7b97\uff0c\u53ea\u80fd\u7528\u4e8e\u526f\u4f5c\u7528\u64cd\u4f5c<\/strong><\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h3 class=\"wp-block-heading\">\u26a0\ufe0f 3\ufe0f\u20e3 \u5916\u90e8\u8fde\u63a5\u4e00\u5b9a\u8981\u5728\u5206\u533a\u5185\u521b\u5efa<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u6570\u636e\u5e93<\/li>\n\n\n\n<li>Redis<\/li>\n\n\n\n<li>Kafka Producer<\/li>\n\n\n\n<li>HTTP \u5ba2\u6237\u7aef<\/li>\n<\/ul>\n\n\n\n<p>\u90fd\u5e94\u653e\u5728 <code>foreachPartition<\/code> \u5185\u90e8\u521d\u59cb\u5316\u3002<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e94\u3001\u603b\u7ed3<\/h2>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th>\u7b97\u5b50<\/th><th>\u4f5c\u7528<\/th><th>\u7279\u70b9<\/th><\/tr><\/thead><tbody><tr><td>count<\/td><td>\u7edf\u8ba1\u5143\u7d20\u6570\u91cf<\/td><td>\u7b80\u5355\u3001\u9ad8\u6548<\/td><\/tr><tr><td>foreachPartition<\/td><td>\u6309\u5206\u533a\u6267\u884c\u903b\u8f91<\/td><td>\u9002\u5408 IO \u64cd\u4f5c<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\">\u4e00\u53e5\u8bdd\u8bb0\u5fc6<\/h3>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<p><strong>count \u7528\u6765\u201c\u6570\u6570\u636e\u201d\uff0cforeachPartition \u7528\u6765\u201c\u5e72\u6d3b\u201d\u3002<\/strong><\/p>\n<\/blockquote>\n","protected":false},"excerpt":{"rendered":"<p>\u4e0b\u9762\u662f\u4e00\u7bc7\u5b8c\u6574\u3001\u53ef\u76f4\u63a5\u53d1\u5e03\u7684\u6280\u672f\u6559\u7a0b\u6587\u7ae0\uff0c\u6807\u9898\u4e0e\u98ce\u683c\u7b26\u5408\u4f60\u4e4b\u524d\u7684 Spark \u7cfb&#8230; <a class=\"more-link\" href=\"https:\/\/www.52runoob.com\/index.php\/2026\/01\/12\/%e5%a4%a7%e6%95%b0%e6%8d%aespark%ef%bc%88%e4%b8%83%e5%8d%81%e4%ba%94%ef%bc%89%ef%bc%9aaction%e8%a1%8c%e5%8a%a8%e7%ae%97%e5%ad%90foreachpartition%e5%92%8ccount%e4%bd%bf%e7%94%a8%e6%a1%88%e4%be%8b\/\">Continue Reading &rarr;<\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[12],"tags":[],"class_list":["post-1333","post","type-post","status-publish","format-standard","hentry","category-12"],"amp_enabled":true,"_links":{"self":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts\/1333","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/comments?post=1333"}],"version-history":[{"count":1,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts\/1333\/revisions"}],"predecessor-version":[{"id":1334,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts\/1333\/revisions\/1334"}],"wp:attachment":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/media?parent=1333"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/categories?post=1333"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/tags?post=1333"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}