java简单爬虫教程

<code class="xml"><groupId>com.example</groupId>
<artifactId>crawler</artifactId>
<version>1.0-SNAPSHOT</version>

<dependencies>
  <dependency>
    <groupId>org.jsoup</groupId>
    <artifactId>jsoup</artifactId>
    <version>1.15.3</version>
  </dependency>
  <dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5.13</version>
  </dependency>
</dependencies></code>

<code class="java">import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

public class Crawler {

  public static void main(String[] args) throws IOException, InterruptedException {
    // 创建 HTTP 客户端
    HttpClient client = HttpClient.newHttpClient();

    // 起始 URL
    String url = "https://example.com";

    // 限制最大并发请求数
    int maxConcurrentRequests = 10;

    // 创建一个异步请求列表
    List<CompletableFuture<Void>> requests = List.of();

    // 递归爬取网页
    crawlRecursively(client, url, maxConcurrentRequests, requests);

    // 等待所有请求完成
    CompletableFuture.allOf(requests).get();
  }

  private static void crawlRecursively(HttpClient client, String url, int maxConcurrentRequests, List<CompletableFuture<Void>> requests)
      throws IOException, InterruptedException {
    // 发送 GET 请求
    HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(url)).build();
    HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

    // 解析 HTML 文档
    Document doc = Jsoup.parse(response.body());

    // 提取页面上的链接
    for (Element link : doc.select("a[href]")) {
      String nextUrl = link.attr("href");

      // 过滤不必要的链接
      if (!nextUrl.startsWith("http") || nextUrl.startsWith("javascript")) {
        continue;
      }

      // 限制并发请求数
      if (requests.size() >= maxConcurrentRequests) {
        CompletableFuture.allOf(requests).get();
        requests.clear();
      }

      // 异步爬取新页面
      requests.add(CompletableFuture.runAsync(() -> {
        try {
          crawlRecursively(client, nextUrl, maxConcurrentRequests, requests);
        } catch (IOException | InterruptedException e) {
          e.printStackTrace();
        }
      }));
    }
  }
}</code>

<code class="bash">mvn clean install
java -jar target/crawler-1.0-SNAPSHOT.jar</code>