{"id":755,"date":"2025-12-12T10:27:53","date_gmt":"2025-12-12T02:27:53","guid":{"rendered":"https:\/\/www.52runoob.com\/?p=755"},"modified":"2025-12-12T10:27:53","modified_gmt":"2025-12-12T02:27:53","slug":"python%e7%88%ac%e8%99%ab%e5%85%a5%e9%97%a8%e3%80%9015%e3%80%91%e7%85%8e%e8%9b%8b%e7%bd%91xxoo%e5%9b%be%e7%89%87%e6%8a%93%e5%8f%96","status":"publish","type":"post","link":"https:\/\/www.52runoob.com\/index.php\/2025\/12\/12\/python%e7%88%ac%e8%99%ab%e5%85%a5%e9%97%a8%e3%80%9015%e3%80%91%e7%85%8e%e8%9b%8b%e7%bd%91xxoo%e5%9b%be%e7%89%87%e6%8a%93%e5%8f%96\/","title":{"rendered":"Python\u722c\u866b\u5165\u95e8\u301015\u3011:\u714e\u86cb\u7f51XXOO\u56fe\u7247\u6293\u53d6"},"content":{"rendered":"\n

Python \u722c\u866b\u5165\u95e8 \u3010\u5341\u4e94\u3011\uff1a\u714e\u86cb\u7f51\uff08Jiandan\uff09\u56fe\u7247\u6293\u53d6 \u6559\u7a0b<\/h1>\n\n\n\n

\u4e0b\u9762\u7ed9\u51fa\u4e00\u5957\u5408\u89c4\u3001\u7a33\u5065\u3001\u53ef\u76f4\u63a5\u8fd0\u884c<\/strong>\u7684\u5165\u95e8\u7ea7\u722c\u866b\u6559\u7a0b\uff08\u4ee5\u6293\u53d6\u7f51\u9875\u4e2d\u7684\u56fe\u7247\u4e3a\u4f8b\uff09\u3002\u5185\u5bb9\u5305\u542b\uff1a\u6ce8\u610f\u4e8b\u9879\uff08\u5408\u89c4\u4e0e\u793c\u8282\uff09\u3001\u5de5\u4f5c\u6d41\u7a0b\u3001\u793a\u4f8b\u4ee3\u7801\uff08\u540c\u6b65\u4e0e\u5e76\u53d1\u4e24\u4e2a\u7248\u672c\uff09\u3001\u5e38\u89c1\u95ee\u9898\u53ca\u5982\u4f55\u4fdd\u5b58\u6587\u4ef6\u3002\u793a\u4f8b\u4ee3\u7801\u5c3d\u91cf\u901a\u7528\uff08\u4e0d\u4f9d\u8d56\u5bf9\u67d0\u4e00\u7ad9\u70b9\u7684\u7279\u5b9a DOM \u7ed3\u6784\uff09\uff0c\u5728\u5b9e\u9645\u4f7f\u7528\u524d\u8bf7\u5148\u68c0\u67e5\u76ee\u6807\u7ad9\u70b9\u7684 robots.txt \u4e0e\u4f7f\u7528\u6761\u6b3e\u5e76\u5c0a\u91cd\u7248\u6743\u4e0e\u9690\u79c1<\/strong>\u3002<\/p>\n\n\n\n

\n\n\n\n

\u4e00\u3001\u91cd\u8981\u6ce8\u610f\u4e8b\u9879\uff08\u5fc5\u987b\u8bfb\uff09<\/h1>\n\n\n\n
    \n
  1. \u5408\u6cd5\u4e0e\u4f26\u7406<\/strong>\uff1a\u5148\u786e\u8ba4\u76ee\u6807\u9875\u9762\u5141\u8bb8\u6293\u53d6\u3002\u4e0d\u8981\u7ed5\u8fc7\u767b\u5f55\u3001\u9a8c\u8bc1\u7801\u6216\u4ed8\u8d39\u5899\u3002\u4e0d\u8981\u6293\u53d6\u672a\u6210\u5e74\u4eba\u76f8\u5173\u6216\u8fdd\u6cd5\/\u4fb5\u72af\u9690\u79c1\u7684\u5185\u5bb9\u3002<\/li>\n\n\n\n
  2. \u5c0a\u91cd robots.txt<\/strong>\uff1a\/robots.txt<\/code>\u00a0\u53ef\u80fd\u9650\u5236\u6293\u53d6\u8def\u5f84\u3002<\/li>\n\n\n\n
  3. \u8bc6\u522b\u5185\u5bb9\u4e0e\u5e74\u9f84\u9650\u5236<\/strong>\uff1a\u82e5\u76ee\u6807\u5305\u542b\u6210\u4eba\u5185\u5bb9\uff0c\u8bf7\u786e\u4fdd\u4f60\u5728\u6cd5\u5f8b\u5141\u8bb8\u7684\u5730\u533a\u4e14\u7528\u4e8e\u5408\u6cd5\u7528\u9014\u3002<\/li>\n\n\n\n
  4. \u793c\u8c8c\u6293\u53d6<\/strong>\uff1a\u8bbe\u7f6e\u5408\u7406\u7684 User-Agent\u3001\u9650\u901f\uff08\u4f8b\u5982\u6bcf\u79d2\u4e00\u4e24\u4e2a\u8bf7\u6c42\u6216\u66f4\u6162\uff09\uff0c\u4f7f\u7528\u91cd\u8bd5\u4e0e\u56de\u9000\u7b56\u7565\uff0c\u907f\u514d\u77ed\u65f6\u95f4\u5185\u5237\u7206\u670d\u52a1\u5668\u3002<\/li>\n\n\n\n
  5. \u5f02\u5e38\u5904\u7406<\/strong>\uff1a\u907f\u514d\u56e0\u4e00\u5f20\u56fe\u7247\u5931\u8d25\u5bfc\u81f4\u7a0b\u5e8f\u5d29\u6e83\uff1b\u505a\u597d\u91cd\u8bd5\u3001\u8d85\u65f6\u3001\u65ad\u70b9\u7eed\u4f20\uff08\u53ef\u9009\uff09\u3002<\/li>\n\n\n\n
  6. \u4e0d\u8981\u516c\u5f00\/\u4f20\u64ad\u4fb5\u6743\u5185\u5bb9<\/strong>\uff1a\u6293\u53d6\u4ec5\u7528\u4e8e\u5b66\u4e60\u6216\u5df2\u83b7\u6388\u6743\u7684\u7528\u9014\u3002<\/li>\n<\/ol>\n\n\n\n
    \n\n\n\n

    \u4e8c\u3001\u6293\u53d6\u5de5\u4f5c\u6d41\u7a0b\uff08\u901a\u7528\uff09<\/h1>\n\n\n\n
      \n
    1. \u8bf7\u6c42\u76ee\u6807\u9875\u9762\uff08\u5e26\u5408\u9002 header\uff0c\u5982 User-Agent\uff09\u3002<\/li>\n\n\n\n
    2. \u89e3\u6790 HTML\uff08\u4f7f\u7528 BeautifulSoup \/ lxml \/ parsel\uff09\u3002<\/li>\n\n\n\n
    3. \u4ece HTML \u4e2d\u63d0\u53d6\u56fe\u7247 URL\uff08\u5e38\u89c1\u4f4d\u7f6e\uff1a<img src=\"\"><\/code>\u3001data-original<\/code>\u3001CSS \u80cc\u666f\u56fe\uff09\u3002<\/li>\n\n\n\n
    4. \u89c4\u8303\u5316 URL\uff08\u8865\u5168\u76f8\u5bf9\u8def\u5f84\u3001\u5904\u7406\u00a0\/\/<\/code>\u00a0\u534f\u8bae\u76f8\u5bf9\u5730\u5740\uff09\u3002<\/li>\n\n\n\n
    5. \u4e0b\u8f7d\u56fe\u7247\u5e76\u4fdd\u5b58\u5230\u672c\u5730\uff08\u4f7f\u7528\u5408\u9002\u6587\u4ef6\u540d\uff0c\u907f\u514d\u91cd\u540d\uff09\u3002<\/li>\n\n\n\n
    6. \u52a0\u5165\u9650\u901f\u3001\u91cd\u8bd5\u3001\u65e5\u5fd7\u4e0e\u5f02\u5e38\u5904\u7406\u3002<\/li>\n<\/ol>\n\n\n\n
      \n\n\n\n

      \u4e09\u3001\u540c\u6b65\u793a\u4f8b\uff08requests + BeautifulSoup\uff09<\/h1>\n\n\n\n

      \u4fdd\u5b58\u4e3a jiandan_images_sync.py<\/code>\uff0c\u5728\u547d\u4ee4\u884c\u8fd0\u884c\uff1apython jiandan_images_sync.py<\/code><\/p>\n\n\n

      \n#!\/usr\/bin\/env python3\n# -*- coding: utf-8 -*-\n\n"""\n\u540c\u6b65\u793a\u4f8b\uff1a\u9002\u5408\u5c0f\u91cf\u56fe\u7247\u6293\u53d6\n\u4f9d\u8d56\uff1arequests, beautifulsoup4\npip install requests beautifulsoup4\n"""\n\nimport os\nimport time\nimport logging\nfrom urllib.parse import urljoin, urlparse\nimport requests\nfrom bs4 import BeautifulSoup\n\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s')\n\nHEADERS = {\n    "User-Agent": "Mozilla\/5.0 (compatible; ImageCrawler\/1.0; +https:\/\/example.com\/bot)"\n}\nTIMEOUT = 15\nRETRY = 3\nSLEEP_BETWEEN_REQUESTS = 1.0  # \u79d2\uff0c\u793c\u8c8c\u722c\u53d6\n\ndef fetch(url):\n    for attempt in range(RETRY):\n        try:\n            resp = requests.get(url, headers=HEADERS, timeout=TIMEOUT)\n            resp.raise_for_status()\n            return resp\n        except Exception as e:\n            logging.warning("fetch failed %s (attempt %d\/%d): %s", url, attempt+1, RETRY, e)\n            time.sleep(1 + attempt*1)\n    return None\n\ndef parse_image_urls(page_url, html):\n    """\n    \u901a\u7528\u89e3\u6790\uff1a\u67e5\u627e &lt;img> \u6807\u7b7e\uff0c\u4ee5\u53ca\u5e38\u89c1\u7684\u5ef6\u8fdf\u52a0\u8f7d\u5c5e\u6027 data-original \/ data-src \u7b49\n    \u8fd4\u56de\uff1a\u53bb\u91cd\u7684\u7edd\u5bf9 URL \u5217\u8868\n    """\n    soup = BeautifulSoup(html, "lxml")\n    imgs = set()\n    for img in soup.find_all("img"):\n        # \u5e38\u89c1\u5c5e\u6027\n        for attr in ("data-original", "data-src", "data-lazy", "src"):\n            src = img.get(attr)\n            if src:\n                # \u8fc7\u6ee4 tiny icons \/ data URIs\n                if src.startswith("data:"):\n                    continue\n                abs_url = urljoin(page_url, src)\n                imgs.add(abs_url)\n                break\n    # \u4e5f\u53ef\u4ee5\u6269\u5c55\u5230 background-image \u7b49\n    return sorted(imgs)\n\ndef make_filename_from_url(img_url, out_dir):\n    parsed = urlparse(img_url)\n    name = os.path.basename(parsed.path)\n    if not name:\n        # fallback\n        name = parsed.netloc.replace('.', '_')\n    # \u9632\u6b62\u6ca1\u6709\u6269\u5c55\u540d\n    if '.' not in name:\n        name += '.jpg'\n    # \u4fdd\u8bc1\u552f\u4e00\n    file_path = os.path.join(out_dir, name)\n    base, ext = os.path.splitext(file_path)\n    counter = 1\n    while os.path.exists(file_path):\n        file_path = f"{base}_{counter}{ext}"\n        counter += 1\n    return file_path\n\ndef download_image(img_url, out_dir):\n    resp = fetch(img_url)\n    if not resp:\n        logging.error("Failed to download image: %s", img_url)\n        return False\n    path = make_filename_from_url(img_url, out_dir)\n    try:\n        with open(path, "wb") as f:\n            f.write(resp.content)\n        logging.info("Saved: %s", path)\n        return True\n    except Exception as e:\n        logging.error("Write failed %s: %s", path, e)\n        return False\n\ndef crawl_page(page_url, out_dir):\n    resp = fetch(page_url)\n    if not resp:\n        logging.error("Failed to fetch page: %s", page_url)\n        return\n    img_urls = parse_image_urls(page_url, resp.text)\n    logging.info("Found %d image urls on %s", len(img_urls), page_url)\n    os.makedirs(out_dir, exist_ok=True)\n    for img_url in img_urls:\n        download_image(img_url, out_dir)\n        time.sleep(SLEEP_BETWEEN_REQUESTS)\n\nif __name__ == "__main__":\n    # \u793a\u4f8b\u5165\u53e3\uff1a\u4f60\u53ef\u4ee5\u628a\u8981\u6293\u53d6\u7684\u9875\u9762 URL \u5217\u8868\u586b\u5728\u8fd9\u91cc\n    pages = [\n        "https:\/\/jandan.net\/ooxx",  # \u793a\u4f8b\u9875\u9762\uff08\u89c6\u5b9e\u9645\u9875\u9762\u800c\u5b9a\uff09\n    ]\n    out_dir = "images"\n    for p in pages:\n        crawl_page(p, out_dir)\n\n<\/pre><\/div>\n\n\n
      \u8bf4\u660e\uff1a<\/p>\n\n\n\n
        \n
      • parse_image_urls<\/code>\u00a0\u5c1d\u8bd5\u8bfb\u53d6\u00a0data-original<\/code>\u3001data-src<\/code>\u3001src<\/code>\u00a0\u7b49\u5c5e\u6027\uff0c\u9002\u5e94\u5ef6\u8fdf\u52a0\u8f7d\u573a\u666f\u3002<\/li>\n\n\n\n
      • SLEEP_BETWEEN_REQUESTS<\/code>\u00a0\u63a7\u5236\u901f\u5ea6\u3002<\/li>\n\n\n\n
      • \u5bf9\u6587\u4ef6\u547d\u540d\u505a\u4e86\u552f\u4e00\u6027\u4fdd\u62a4\u3002<\/li>\n<\/ul>\n\n\n\n
        \n\n\n\n
        \u56db\u3001\u5e76\u53d1\u793a\u4f8b\uff08aiohttp + asyncio\uff09\u2014\u2014\u9002\u5408\u5927\u91cf\u56fe\u7247\u4f46\u5bf9\u76ee\u6807\u7ad9\u70b9\u66f4\u53cb\u597d\u9700\u66f4\u8c28\u614e<\/h1>\n\n\n\n
        \u4fdd\u5b58\u4e3a jiandan_images_async.py<\/code>\uff0c\u4f9d\u8d56\uff1aaiohttp<\/code>\u3001aiofiles<\/code>\u3001beautifulsoup4<\/code>\u3001async_timeout<\/code>\u3002
        \u5b89\u88c5\uff1apip install aiohttp aiofiles beautifulsoup4 async-timeout<\/code><\/p>\n\n\n
        \n#!\/usr\/bin\/env python3\n# -*- coding: utf-8 -*-\n\nimport asyncio\nimport aiohttp\nimport aiofiles\nimport async_timeout\nimport os\nfrom urllib.parse import urljoin, urlparse\nfrom bs4 import BeautifulSoup\nimport logging\nimport time\n\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s')\n\nHEADERS = {"User-Agent": "Mozilla\/5.0 (compatible; AsyncImageCrawler\/1.0)"}\nCONCURRENCY = 8\nTIMEOUT = 15\nRETRY = 3\n\nasync def fetch_text(session, url):\n    for attempt in range(RETRY):\n        try:\n            with async_timeout.timeout(TIMEOUT):\n                async with session.get(url, headers=HEADERS) as resp:\n                    resp.raise_for_status()\n                    return await resp.text()\n        except Exception as e:\n            logging.warning("fetch_text failed %s attempt %d: %s", url, attempt+1, e)\n            await asyncio.sleep(1 + attempt)\n    return None\n\ndef parse_image_urls(page_url, html):\n    soup = BeautifulSoup(html, "lxml")\n    imgs = set()\n    for img in soup.find_all("img"):\n        for attr in ("data-original", "data-src", "data-lazy", "src"):\n            src = img.get(attr)\n            if src and not src.startswith("data:"):\n                imgs.add(urljoin(page_url, src))\n                break\n    return sorted(imgs)\n\ndef make_filename_from_url(img_url, out_dir):\n    parsed = urlparse(img_url)\n    name = os.path.basename(parsed.path) or parsed.netloc.replace('.', '_')\n    if '.' not in name:\n        name += '.jpg'\n    file_path = os.path.join(out_dir, name)\n    base, ext = os.path.splitext(file_path)\n    counter = 1\n    while os.path.exists(file_path):\n        file_path = f"{base}_{counter}{ext}"\n        counter += 1\n    return file_path\n\nasync def download_image(session, img_url, out_dir, sem):\n    async with sem:\n        for attempt in range(RETRY):\n            try:\n                with async_timeout.timeout(TIMEOUT):\n                    async with session.get(img_url, headers=HEADERS) as resp:\n                        resp.raise_for_status()\n                        content = await resp.read()\n                        path = make_filename_from_url(img_url, out_dir)\n                        async with aiofiles.open(path, 'wb') as f:\n                            await f.write(content)\n                        logging.info("Saved: %s", path)\n                        return True\n            except Exception as e:\n                logging.warning("download failed %s attempt %d: %s", img_url, attempt+1, e)\n                await asyncio.sleep(1 + attempt)\n        logging.error("Give up downloading %s", img_url)\n        return False\n\nasync def main(pages, out_dir):\n    os.makedirs(out_dir, exist_ok=True)\n    sem = asyncio.Semaphore(CONCURRENCY)\n    async with aiohttp.ClientSession() as session:\n        for page in pages:\n            html = await fetch_text(session, page)\n            if not html:\n                logging.error("Cannot fetch %s", page)\n                continue\n            img_urls = parse_image_urls(page, html)\n            tasks = [download_image(session, u, out_dir, sem) for u in img_urls]\n            # \u5e76\u53d1\u4e0b\u8f7d\u5f53\u524d\u9875\u9762\u56fe\u7247\n            await asyncio.gather(*tasks)\n            # \u793c\u8c8c\u7b49\u5f85\n            await asyncio.sleep(1)\n\nif __name__ == "__main__":\n    pages = ["https:\/\/jandan.net\/ooxx"]\n    out_dir = "images_async"\n    asyncio.run(main(pages, out_dir))\n\n<\/pre><\/div>\n\n\n
        \u8bf4\u660e\uff1a<\/p>\n\n\n\n