基于 express 大文件切片上传

本文由简悦 SimpRead 转码，原文地址 zhuanlan.zhihu.com

本文代码已上传 Gitee：带只拖鞋去流浪 / 基于 express 大文件上传 dev 分支

优化代码：git checkout dev2 git pull origin dev2

思路：假如你要搬一个组装式的大桌子，桌子太大只能拆解成一块块搬过去再组装起来。

创建 express 服务
前端
服务端
切片优化
并发量优化
断点续传

一、创建 express 服务

I. 初始化

mkdir test && cd test
npm init -y
yarn add express
yarn add nodemon      # node 热更新 或者使用 supervisor (yarn add supervisor)
yarn add cli-color    # 输出命令行颜色

II. 创建服务

const express = require('express');
const clc = require('cli-color');

const server = express();

server.get('/', (req, res) => {
    res.send('hi,express.');
});

server.listen(3000, _ => {
    console.log(clc.bold.blue.underline('http://localhost:3000/'));
    console.log(clc.bold.blue.underline('http://127.0.0.1:3000/'));
});

III. 配置启动

a. 如果使用 nodemon

// package.json
{
  "scripts": {
    "serve": "nodemon ./src/index.js"
  }
}

b. 如果使用 supervisor

将上面的 nodemon 改成 supervisor 就可以了。

IV. 测试

二、前端

I. HTML

<div id="app">
    <form action="">
        <input type="file" >
        <button id="uploadBtn">上传</button>
    </form>
</div>

II. 绑定事件

// 文件被更改
function handleFileChange(event) { ... }

// 大文件上传
async function handleFileUpload(event) { ... }

document.getElementById('uploadInput').addEventListener('change', handleFileChange);
document.getElementById('uploadBtn').addEventListener('click', handleFileUpload);

III. 文件被更改 handleFileChange

var file = null;

// 文件被更改
function handleFileChange(event) {
    const file = event.target.files[0];
    if (!file) return;
    window.file = file;
}

IV. 大文件上传

// 大文件上传
async function handleFileUpload(event) {
    event.preventDefault();

    const file = window.file;
    if (!file) return;

    // 创建切片
    const createFileChunks = function (file, size = SIZE) { ... }

    // 上传切片
    const uploadFileChunks = async function (fileChunks, filename) { ... }

    // 合并切片
    const mergeFileChunks = async function (filename) { ... }

    const fileChunks = createFileChunks(file);
    await uploadFileChunks(fileChunks, file.name);
    await mergeFileChunks(file.name);

    console.log('上传完成');
}

V. 创建切片

const SIZE = 1024 * 1024 * 10; // 10MB 切片大小
 
// 创建切片
const createFileChunks = function (file, size = SIZE) {
    let fileChunks = [];
    for(let cur = 0; cur < file.size; cur += size){
        fileChunks.push(file.slice(cur, cur + size));
    }
    return fileChunks;
}

VI. 上传切片

a. 引入 axios（CDN）

<script src="https://unpkg.com/axios/dist/axios.min.js"></script>

b. 配置 axios.defaults.baseURL

axios.defaults.baseURL = `http://localhost:3000`;

c. uploadFileChunks

// 上传切片
const uploadFileChunks = async function (fileChunks, filename) {

    const chunksList = fileChunks.map((chunk, index) => {
        let formData = new FormData();
        formData.append('filename', filename);
        formData.append('hash', index);
        formData.append('chunk', chunk);
        return {
            formData
        };
    });

    const uploadList = chunksList.map(({
        formData
    }) => axios({
        method: 'post',
        url: '/upload',
        data: formData
    }));

    await Promise.all(uploadList);
}

VII. 合并切片

// 合并切片
const mergeFileChunks = async function (filename) {
    await axios({
        method: 'get',
        url: '/merge',
        params: {
            filename
        }
    });
}

三、服务端

I. 切片上传 server:post('/upload')

a. multiparty

b. events 事件触发器 | Node.js API 文档

c. fs 文件系统 | Node.js API 文档

d. stream 流 | Node.js API 文档

const multiparty = require('multiparty');
const EventEmitter = require('events');
const fs = require('fs');
const path = require('path');

const STATIC_TEMPORARY = path.resolve(__dirname, '../static/temporary');

server.post('/upload', (req, res) => {

    const multipart = new multiparty.Form();
    const myEmitter = new EventEmitter();

    const formData = {
        filename: undefined,
        hash: undefined,
        chunk: undefined,
    }

    let isFieldOk = false, isFileOk = false;

    multipart.parse(req, function (err, fields, files) {
        formData.filename = fields['filename'][0];
        formData.hash = fields['hash'][0];

        isFieldOk = true;
        myEmitter.emit('start');
    });

    multipart.on('file', function (name, file) {
        formData.chunk = file;
        isFileOk = true;
        myEmitter.emit('start');
    });

    myEmitter.on('start', function () {
        if (isFieldOk && isFileOk) {
            const { filename, hash, chunk } = formData;
            const dir = `${STATIC_TEMPORARY}/${filename}`;

            try {
                if (!fs.existsSync(dir)) fs.mkdirSync(dir);

                const buffer = fs.readFileSync(chunk.path);
                const ws = fs.createWriteStream(`${dir}/${hash}`);
                ws.write(buffer);
                ws.close();

                res.send(`${filename}-${hash} 切片上传成功`)
            } catch (error) {
                console.error(error);
            }

            isFieldOk = false;
            isFileOk = false;

        }
    });
});

II. 合并切片

const { Buffer } = require('buffer');
const STATIC_FILES = path.resolve(__dirname, '../static/files');

server.get('/merge', async (req, res) => {
    const { filename } = req.query;

    try {
        let len = 0;
        const bufferList = fs.readdirSync(`${STATIC_TEMPORARY}/${filename}`).map(hash => {
            const buffer = fs.readFileSync(`${STATIC_TEMPORARY}/${filename}/${hash}`);
            len += buffer.length;
            return buffer;
        });

        const buffer = Buffer.concat(bufferList, len);
        const ws = fs.createWriteStream(`${STATIC_FILES}/${filename}`);
        ws.write(buffer);
        ws.close();

        res.send(`切片合并完成`);
    } catch (error) {
        console.error(error);
    }
})

最后，切片合并完成之后需要删除 temporary 临时存放的切片，删除整个文件夹

function deleteFolder(filepath) {
    if (fs.existsSync(filepath)) {
        fs.readdirSync(filepath).forEach(filename => {
            const fp = `${filepath}/${filename}`;
            if (fs.statSync(fp).isDirectory()) deleteFolder(fp);
            else fs.unlinkSync(fp);
        });
        fs.rmdirSync(filepath);
    }
}

四、切片优化

分治思想，递归切片

// 递归切片（切出来不全都是 SIZE）
function createFileChunks(file, size = SIZE) {
-   // const fileChunks = [];
-   // for (let cur = 0; cur < file.size; cur += size)
-   //    fileChunks.push(file.slice(cur, cur + size));
-   // return fileChunks;
    
+   const count = file.size;
+   if(count <= size) return [file];
+   const mid = count / 2;
+   const leftChunk = fileChunk.slice(0, mid);
+   const rightChunk = fileChunk.slice(mid);
+   return [...createFileChunks(leftChunk), ...createFileChunks(rightChunk)];
}

五、并发量优化

每次并发请求数量控制在 10 次以内

// 上传切片
const uploadFileChunks = async function (fileChunks, filename) {

+   for (let i = 0, len = axiosList.length; i < len; i += 10) {
+       const list = i + 10 < len ? axiosList.slice(i, i + 10) : axiosList.slice(i);
+       await Promise.all(list);
+   }

-   // await Promise.all(axiosList);
}

六、断点续传

递归续传

let failUploadDict = {};

// 上传切片
const uploadFileChunks = async function (formDataList, failUploadDict = {}) {
    if (!formDataList.length) return;

    const axiosList = formDataList.map(({
        formData
    }) => {
        const index = formData.get('index');
        return axios({
            method: 'post',
            url: `/upload`,
            data: formData
        }).then(res => {
            console.log(res.data);
            delete failUploadDict[index];
        }).catch(err => {
            console.error(err);
            failUploadDict[index] = formData;
        })
    });

    for (let i = 0, len = axiosList.length; i < len; i += 10) {
        const list = i + 10 < len ? axiosList.slice(i, i + 10) : axiosList.slice(i);
        await Promise.all(list);
    }

    // await Promise.all(axiosList);

    formDataList = Object.values(failUploadDict), failUploadDict = {};
    return uploadFileChunks(formDataList);
}

带只拖鞋去流浪的个人空间_哔哩哔哩_Bilibili 带只拖鞋去流浪的博客_CSDN 博客 - 渗透测试, java 基础, leetcode 领域博主带只拖鞋去流浪 - 简书www.jianshu.com/u/45339cbb7573

警告

商业转载请联系本人，非商业转载请注明出处。

如果觉得文章对你有用，请随意赞赏

基于 express 大文件切片上传

https://blog.skybluefeet.cn/archives/%E5%9F%BA%E4%BA%8E-express-%E5%A4%A7%E6%96%87%E4%BB%B6%E5%88%87%E7%89%87%E4%B8%8A%E4%BC%A0

作者

天空那片蓝

发布于

2025-08-24

更新于

2025-08-24

许可协议

CC BY 4.0