查询域名在archive.org wayback是否有存档

网络技术 3 浏览 2 分钟阅读

四、代码


import requests
import csv
import time
from datetime import datetime
from urllib.parse import urlparse

API = "https://web.archive.org/cdx/search/cdx"

def check_wayback(domain):
    domain = domain.strip().lower()
    params = {
        "url": domain,
        "matchType": "domain",
        "output": "json",
        "fl": "timestamp,original,statuscode",
        "limit": "10000"
    }

    for attempt in range(3):
        try:
            r = requests.get(
                API,
                params=params,
                timeout=60,
                headers={"User-Agent": "WaybackDomainChecker/1.0"}
            )

            if r.status_code != 200:
                raise Exception(f"HTTP {r.status_code}")

            data = r.json()

            if not data:
                return [domain, "No", "", "", 0, "OK"]

            # 第一行是字段名称
            records = data[1:] if len(data) > 1 else []

            if not records:
                return [domain, "No", "", "", 0, "OK"]

            dates = sorted(row[0] for row in records)

            first = dates[0]
            last = dates[-1]

            return [
                domain,
                "Yes",
                first[:8],
                last[:8],
                len(records),
                "OK"
            ]

        except Exception as e:
            if attempt == 2:
                return [domain, "Unknown", "", "", 0, str(e)]

            time.sleep(5 * (attempt + 1))

    return [domain, "Unknown", "", "", 0, "Error"]


with open("domains.txt", "r", encoding="utf-8") as f:
    domains = list(dict.fromkeys(
        line.strip() for line in f if line.strip()
    ))

results = []

for i, domain in enumerate(domains, 1):
    result = check_wayback(domain)
    results.append(result)

    print(f"[{i}/{len(domains)}] {domain}: {result[1]}")
    time.sleep(1)

with open(
    "wayback_results.csv",
    "w",
    newline="",
    encoding="utf-8-sig"
) as f:
    writer = csv.writer(f)
    writer.writerow([
        "Domain", "Wayback", "First Archive",
        "Last Archive", "Archive Count", "Status"
    ])
    writer.writerows(results)

print("\nFinished! Results saved to wayback_results.csv")

相关文章