{"id":430,"date":"2025-12-04T14:57:24","date_gmt":"2025-12-04T06:57:24","guid":{"rendered":"https:\/\/www.52runoob.com\/?p=430"},"modified":"2025-12-04T14:57:24","modified_gmt":"2025-12-04T06:57:24","slug":"%e8%ae%ba%e5%9d%9b%e6%95%b0%e6%8d%ae%e7%88%ac%e8%99%ab-%e7%88%ac%e5%8f%96%e8%ae%ba%e5%9d%9b%e5%b8%96%e5%ad%90%e5%92%8c%e7%94%a8%e6%88%b7%e8%a1%8c%e4%b8%ba%e6%95%b0%e6%8d%ae%e8%bf%9b%e8%a1%8c","status":"publish","type":"post","link":"https:\/\/www.52runoob.com\/index.php\/2025\/12\/04\/%e8%ae%ba%e5%9d%9b%e6%95%b0%e6%8d%ae%e7%88%ac%e8%99%ab-%e7%88%ac%e5%8f%96%e8%ae%ba%e5%9d%9b%e5%b8%96%e5%ad%90%e5%92%8c%e7%94%a8%e6%88%b7%e8%a1%8c%e4%b8%ba%e6%95%b0%e6%8d%ae%e8%bf%9b%e8%a1%8c\/","title":{"rendered":"\u8bba\u575b\u6570\u636e\u722c\u866b – \u722c\u53d6\u8bba\u575b\u5e16\u5b50\u548c\u7528\u6237\u884c\u4e3a\u6570\u636e,\u8fdb\u884c\u5206\u6790"},"content":{"rendered":"\n

\u4e0b\u9762\u7ed9\u4f60\u4e00\u4efd \u300a\u8bba\u575b\u6570\u636e\u722c\u866b\uff1a\u4ece\u5e16\u5b50\u5230\u7528\u6237\u884c\u4e3a\u6570\u636e\u7684\u5b8c\u6574\u91c7\u96c6\u4e0e\u5206\u6790\u6559\u5b66\u300b<\/strong>\uff0c\u9002\u7528\u4e8e Python\uff08Requests + BeautifulSoup \/ Selenium\uff09+ \u6570\u636e\u5206\u6790\uff08Pandas + \u53ef\u89c6\u5316\uff09\u3002\u5185\u5bb9\u5305\u542b\u6574\u4f53\u67b6\u6784\u3001\u53cd\u722c\u7ed5\u8fc7\u3001\u6570\u636e\u5e93\u5efa\u6a21\u4e0e\u5206\u6790\u6d41\u7a0b\u3002\u4f60\u53ef\u4ee5\u76f4\u63a5\u7528\u5728\u5b9e\u9645\u9879\u76ee\u91cc\u3002<\/p>\n\n\n\n

\n\n\n\n

\u2705 \u8bba\u575b\u6570\u636e\u722c\u866b\uff1a\u722c\u53d6\u5e16\u5b50 + \u7528\u6237\u884c\u4e3a\u6570\u636e\uff08\u8d85\u8be6\u7ec6\u6559\u7a0b\uff09<\/strong><\/h1>\n\n\n\n
\n

\u9002\u7528\uff1aDiscuz \/ phpBB \/ \u8d34\u5427 \/ \u7b80\u6613\u8bba\u575b\u7cfb\u7edf
\u6280\u672f\u6808\uff1aPython\u3001Requests\u3001BeautifulSoup\u3001lxml\u3001Selenium\uff08\u53ef\u9009\uff09\u3001Pandas\u3001MySQL\/MongoDB<\/p>\n<\/blockquote>\n\n\n\n

\n\n\n\n

\ud83d\udd25 \u4e00\u3001\u9879\u76ee\u76ee\u6807<\/strong><\/h1>\n\n\n\n

\u4f60\u5c06\u80fd\u5b9e\u73b0\u5982\u4e0b\u529f\u80fd\uff1a<\/p>\n\n\n\n

\u2714 1. \u722c\u53d6\u8bba\u575b\u677f\u5757\u5217\u8868<\/h3>\n\n\n\n
    \n
  • \u677f\u5757\u540d\u79f0<\/li>\n\n\n\n
  • \u677f\u5757 URL<\/li>\n\n\n\n
  • \u5e16\u5b50\u6570\u91cf<\/li>\n\n\n\n
  • \u6700\u65b0\u53d1\u5e16\u65f6\u95f4<\/li>\n<\/ul>\n\n\n\n

    \u2714 2. \u722c\u53d6\u5e16\u5b50\u5217\u8868\uff08\u5206\u9875\uff09<\/h3>\n\n\n\n
      \n
    • \u5e16\u5b50\u6807\u9898<\/li>\n\n\n\n
    • \u5e16\u5b50 URL<\/li>\n\n\n\n
    • \u53d1\u5e16\u7528\u6237<\/li>\n\n\n\n
    • \u56de\u590d\u6570 \/ \u6d4f\u89c8\u6570<\/li>\n\n\n\n
    • \u53d1\u5e03\u65f6\u95f4<\/li>\n<\/ul>\n\n\n\n

      \u2714 3. \u722c\u53d6\u5e16\u5b50\u8be6\u60c5\uff08\u591a\u9875\uff09<\/h3>\n\n\n\n
        \n
      • \u6bcf\u5c42\u56de\u590d\u5185\u5bb9<\/li>\n\n\n\n
      • \u56de\u590d\u7528\u6237<\/li>\n\n\n\n
      • \u56de\u590d\u65f6\u95f4<\/li>\n\n\n\n
      • \u70b9\u8d5e\u6570\uff08\u5982\u6709\uff09<\/li>\n<\/ul>\n\n\n\n

        \u2714 4. \u7528\u6237\u884c\u4e3a\u6570\u636e<\/h3>\n\n\n\n
          \n
        • \u7528\u6237\u4e2a\u4eba\u8d44\u6599<\/li>\n\n\n\n
        • \u7528\u6237\u53d1\u5e16\u6570\u91cf<\/li>\n\n\n\n
        • \u7528\u6237\u56de\u590d\u6570\u91cf<\/li>\n\n\n\n
        • \u7528\u6237\u6d3b\u8dc3\u65f6\u95f4<\/li>\n\n\n\n
        • \u7528\u6237\u884c\u4e3a\u8def\u5f84\uff08\u53d6\u51b3\u4e8e\u8bba\u575b\u662f\u5426\u63d0\u4f9b\uff09<\/li>\n<\/ul>\n\n\n\n

          \u2714 5. \u6570\u636e\u843d\u5e93<\/h3>\n\n\n\n
            \n
          • MySQL\uff1a\u7ed3\u6784\u5316\u6570\u636e<\/li>\n\n\n\n
          • MongoDB\uff1a\u534a\u7ed3\u6784\u5316\u6587\u672c\u5185\u5bb9<\/li>\n<\/ul>\n\n\n\n

            \u2714 6. \u5206\u6790\u5185\u5bb9<\/h3>\n\n\n\n
              \n
            • \u7528\u6237\u6d3b\u8dc3\u5ea6<\/li>\n\n\n\n
            • \u9ad8\u9891\u7528\u6237<\/li>\n\n\n\n
            • \u5e16\u5b50\u70ed\u5ea6<\/li>\n\n\n\n
            • \u5173\u952e\u8bcd\u63d0\u53d6\uff08NLP\uff09<\/li>\n\n\n\n
            • \u60c5\u611f\u5206\u6790\uff08\u6b63\u9762\/\u8d1f\u9762\uff09<\/li>\n<\/ul>\n\n\n\n
              \n\n\n\n

              \ud83d\ude80 \u4e8c\u3001\u9879\u76ee\u6574\u4f53\u67b6\u6784\u8bbe\u8ba1<\/h1>\n\n\n
              \ncrawler\/\n \u251c\u2500\u2500 main.py\n \u251c\u2500\u2500 config.py\n \u251c\u2500\u2500 forum_spider.py\n \u251c\u2500\u2500 detail_spider.py\n \u251c\u2500\u2500 user_spider.py\n \u251c\u2500\u2500 utils\/\n \u2502    \u251c\u2500\u2500 headers.py\n \u2502    \u251c\u2500\u2500 proxy.py\n \u2502    \u251c\u2500\u2500 db.py\n \u2502    \u2514\u2500\u2500 logger.py\n \u2514\u2500\u2500 data\/\n      \u251c\u2500\u2500 posts.csv\n      \u251c\u2500\u2500 replies.csv\n      \u2514\u2500\u2500 users.csv\n\n<\/pre><\/div>\n\n\n
              \n\n\n\n
              \ud83d\udd27 \u4e09\u3001\u51c6\u5907\u5de5\u4f5c<\/h1>\n\n\n\n
              1. \u5b89\u88c5\u4f9d\u8d56<\/strong><\/h3>\n\n\n
              \npip install requests beautifulsoup4 lxml selenium pandas pymysql\n\n<\/pre><\/div>\n\n\n
              2. \u53ef\u9009\uff1a\u4f7f\u7528 ChromeDriver<\/strong><\/h3>\n\n\n\n
              \u7528\u4e8e\u52a8\u6001\u52a0\u8f7d\u8bba\u575b\uff08\u82e5\u6709\u9a8c\u8bc1\u7801\u3001JS \u5206\u9875\uff09\uff1a<\/p>\n\n\n
              \nfrom selenium import webdriver\n\n<\/pre><\/div>\n\n\n
              \n\n\n\n
              \ud83d\udd78 \u56db\u3001\u722c\u53d6\u6d41\u7a0b\uff08\u542b\u4ee3\u7801\u793a\u4f8b\uff09<\/h1>\n\n\n\n
              \n\n\n\n
              \u6b65\u9aa4 1\uff1a\u722c\u53d6\u677f\u5757\u5217\u8868<\/strong><\/h2>\n\n\n
              \nimport requests\nfrom bs4 import BeautifulSoup\n\nbase_url = "https:\/\/example-forum.com"\n\ndef get_forum_sections():\n    html = requests.get(base_url).text\n    soup = BeautifulSoup(html, "lxml")\n\n    sections = []\n    for sec in soup.select(".forum-section"):\n        sections.append({\n            "name": sec.text.strip(),\n            "url": base_url + sec.get("href")\n        })\n    return sections\n\n<\/pre><\/div>\n\n\n
              \n\n\n\n
              \u6b65\u9aa4 2\uff1a\u722c\u53d6\u5e16\u5b50\u5217\u8868\uff08\u542b\u5206\u9875\uff09<\/strong><\/h2>\n\n\n
              \ndef get_thread_list(section_url):\n    threads = []\n\n    page = 1\n    while True:\n        url = f"{section_url}&amp;page={page}"\n        html = requests.get(url).text\n        soup = BeautifulSoup(html, "lxml")\n\n        items = soup.select(".thread-item")\n        if not items:\n            break\n\n        for item in items:\n            threads.append({\n                "title": item.select_one(".title").text,\n                "url": base_url + item.select_one(".title a").get("href"),\n                "author": item.select_one(".author").text,\n                "reply_count": item.select_one(".reply").text\n            })\n\n        page += 1\n\n    return threads\n\n<\/pre><\/div>\n\n\n
              \n\n\n\n
              \u6b65\u9aa4 3\uff1a\u722c\u53d6\u5e16\u5b50\u8be6\u60c5\uff08\u591a\u9875\u56de\u590d\uff09<\/strong><\/h2>\n\n\n
              \ndef get_thread_detail(thread_url):\n    page = 1\n    replies = []\n\n    while True:\n        url = f"{thread_url}&amp;page={page}"\n        html = requests.get(url).text\n        soup = BeautifulSoup(html, "lxml")\n\n        reply_items = soup.select(".reply-item")\n        if not reply_items:\n            break\n\n        for r in reply_items:\n            replies.append({\n                "user": r.select_one(".username").text,\n                "content": r.select_one(".reply-content").text.strip(),\n                "time": r.select_one(".reply-time").text\n            })\n\n        page += 1\n\n    return replies\n\n<\/pre><\/div>\n\n\n
              \n\n\n\n
              \u6b65\u9aa4 4\uff1a\u722c\u53d6\u7528\u6237\u6570\u636e<\/strong><\/h2>\n\n\n
              \ndef get_user_info(user_url):\n    html = requests.get(user_url).text\n    soup = BeautifulSoup(html, "lxml")\n\n    return {\n        "name": soup.select_one(".username").text,\n        "posts": soup.select_one(".post-count").text,\n        "replies": soup.select_one(".reply-count").text,\n        "register_time": soup.select_one(".register-time").text\n    }\n\n<\/pre><\/div>\n\n\n
              \n\n\n\n
              \ud83d\uddc4 \u4e94\u3001\u6570\u636e\u5e93\u8868\u7ed3\u6784\uff08MySQL\uff09<\/h1>\n\n\n\n
              user<\/strong><\/h3>\n\n\n\n
              \u5b57\u6bb5<\/th>\u7c7b\u578b<\/th>\u8bf4\u660e<\/th><\/tr><\/thead>
              id<\/td>int<\/td>\u7528\u6237ID<\/td><\/tr>
              username<\/td>varchar<\/td>\u6635\u79f0<\/td><\/tr>
              posts<\/td>int<\/td>\u53d1\u5e16\u6570<\/td><\/tr>
              replies<\/td>int<\/td>\u56de\u590d\u6570<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n
              post<\/strong><\/h3>\n\n\n\n
              \u5b57\u6bb5<\/th>\u7c7b\u578b<\/th>\u8bf4\u660e<\/th><\/tr><\/thead>
              id<\/td>int<\/td>\u5e16\u5b50ID<\/td><\/tr>
              title<\/td>varchar<\/td>\u6807\u9898<\/td><\/tr>
              author<\/td>varchar<\/td>\u4f5c\u8005<\/td><\/tr>
              reply_count<\/td>int<\/td>\u56de\u590d\u6570<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n
              reply<\/strong><\/h3>\n\n\n\n
              \u5b57\u6bb5<\/th>\u7c7b\u578b<\/th>\u8bf4\u660e<\/th><\/tr><\/thead>
              id<\/td>int<\/td>\u56de\u590dID<\/td><\/tr>
              post_id<\/td>int<\/td>\u6240\u5c5e\u5e16\u5b50<\/td><\/tr>
              user<\/td>varchar<\/td>\u7528\u6237\u540d<\/td><\/tr>
              content<\/td>text<\/td>\u5185\u5bb9<\/td><\/tr>
              time<\/td>datetime<\/td>\u65f6\u95f4<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n
              \n\n\n\n
              \ud83d\udcca \u516d\u3001\u6570\u636e\u5206\u6790\uff08Pandas\uff09<\/h1>\n\n\n\n
              1. \u7528\u6237\u6d3b\u8dc3\u5ea6<\/h3>\n\n\n
              \nimport pandas as pd\nusers = pd.read_csv("users.csv")\nprint(users.sort_values("posts", ascending=False).head())\n\n<\/pre><\/div>\n\n\n
              2. \u5e16\u5b50\u70ed\u5ea6\u5206\u6790<\/h3>\n\n\n
              \nposts = pd.read_csv("posts.csv")\nposts["heat"] = posts["reply_count"] * 2\nprint(posts.sort_values("heat", ascending=False).head())\n\n<\/pre><\/div>\n\n\n
              3. NLP \u60c5\u611f\u5206\u6790<\/h3>\n\n\n\n
              \uff08\u53ef\u7528\u4f60\u9879\u76ee\u4e2d\u6b63\u5728\u505a\u7684\u60c5\u611f\u6a21\u578b\uff09<\/p>\n\n\n\n
              \n\n\n\n
              \ud83d\udee1 \u4e03\u3001\u53cd\u722c\u63aa\u65bd\u4e0e\u7ed5\u8fc7\u7b56\u7565<\/h1>\n\n\n\n
              \u53cd\u722c\u624b\u6bb5<\/th>\u5bf9\u7b56<\/th><\/tr><\/thead>
              User-Agent \u9650\u5236<\/td>\u4f2a\u88c5 UA<\/td><\/tr>
              Cookie \u6821\u9a8c<\/td>\u4fdd\u5b58 session<\/td><\/tr>
              \u8bbf\u95ee\u592a\u5feb<\/td>time.sleep \/ \u968f\u673a\u5ef6\u8fdf<\/td><\/tr>
              \u767b\u5f55\u9a8c\u8bc1<\/td>Selenium + \u81ea\u52a8\u767b\u5f55<\/td><\/tr>
              IP \u9650\u5236<\/td>\u4ee3\u7406\u6c60<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n
              \n\n\n\n
              \ud83c\udf81 \u516b\u3001\u53ef\u76f4\u63a5\u8fd0\u884c\u7684\u4e3b\u7a0b\u5e8f\uff08\u6574\u5408\u7248\uff09<\/h1>\n\n\n
              \nfrom forum_spider import get_forum_sections, get_thread_list, get_thread_detail\n\nif __name__ == "__main__":\n    sections = get_forum_sections()\n\n    for sec in sections:\n        threads = get_thread_list(sec["url"])\n        for t in threads:\n            replies = get_thread_detail(t["url"])\n            print("\u6293\u5230\u5e16\u5b50\uff1a", t["title"], "\u56de\u590d\u6570\uff1a", len(replies))\n\n<\/pre><\/div>","protected":false},"excerpt":{"rendered":"
              \u4e0b\u9762\u7ed9\u4f60\u4e00\u4efd \u300a\u8bba\u575b\u6570\u636e\u722c\u866b\uff1a\u4ece\u5e16\u5b50\u5230\u7528\u6237\u884c\u4e3a\u6570\u636e\u7684\u5b8c\u6574\u91c7\u96c6\u4e0e\u5206\u6790\u6559\u5b66\u300b\uff0c\u9002\u7528\u4e8e… Continue Reading →<\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[12],"tags":[],"class_list":["post-430","post","type-post","status-publish","format-standard","hentry","category-12"],"amp_enabled":true,"_links":{"self":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts\/430","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/comments?post=430"}],"version-history":[{"count":1,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts\/430\/revisions"}],"predecessor-version":[{"id":431,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/posts\/430\/revisions\/431"}],"wp:attachment":[{"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/media?parent=430"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/categories?post=430"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.52runoob.com\/index.php\/wp-json\/wp\/v2\/tags?post=430"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}