Java多线程高效读取大文件技巧

AsynchronousFileChannel channel = AsynchronousFileChannel.open(Paths.get("largefile.txt"), StandardOpenOption.READ);
<p>ByteBuffer buffer = ByteBuffer.allocate(8192);
Future<integer> result = channel.read(buffer, 0);</integer></p><p>// 非阻塞：可以继续做其他事
while (!result.isDone()) {
// 可执行其他任务
}</p><p>Integer bytesRead = result.get();</p>

channel.read(buffer, 0, buffer, new CompletionHandler<Integer, ByteBuffer>() {
    public void completed(Integer result, ByteBuffer attachment) {
        if (result == -1) {
            System.out.println("读取完成");
            return;
        }
        attachment.flip();
        // 处理数据...
        attachment.clear();
    }
<pre class="brush:java;toolbar:false;">public void failed(Throwable exc, ByteBuffer attachment) {
    exc.printStackTrace();
}

});

int corePoolSize = Runtime.getRuntime().availableProcessors() + 1;
ExecutorService executor = new ThreadPoolExecutor(
    corePoolSize,
    corePoolSize * 2,
    60L,
    TimeUnit.SECONDS,
    new LinkedBlockingQueue<>(100),
    new ThreadPoolTaskDecorator()
);

long chunkSize = 1024 * 1024;
long fileSize = channel.size();
<p>for (long pos = 0; pos < fileSize; pos += chunkSize) {
long finalPos = pos;
executor.submit(() -> {
ByteBuffer buf = ByteBuffer.allocate((int) Math.min(chunkSize, fileSize - finalPos));
Future<integer> readOp = channel.read(buf, finalPos);
Integer n = readOp.get();
buf.flip();
// 处理该块数据
return n;
});
}</integer></p>

CompletableFuture.supplyAsync(() -> {
    // 异步读取某一块
    return readFileChunk(channel, position, size);
}, executor)
.thenApplyAsync(data -> processChunk(data), executor)
.thenAccept(result -> saveResult(result));