在客户端中安全地上传文件

本文介绍如何用预签名 URL 安全地让客户端端具备高性能的大文件上传能力。

为何不建议直接用 Javascript SDK 等客户端 SDK 来上传文件？

1. 容易被反编译后暴露 ak / sk
2. 不够灵活

安全建议：永远不要使用 API 下发 ak / sk

更好的方案

后端生成「预签名链接」并通过 API 下发

前端可以对预签名链接直接发起 PUT 请求，这样我们可以收获的好处有：

1. 更安全：永远不会因 暴露客户端代码 或 被反编译 而泄露至关重要的 AccessKey 和 SecretKey；
2. 更灵活：利用显式指定 Content-Type 和 Content-Length 来防止客户端被破解，从而绕过业务策略的限制
3. 低成本：统一由服务端传递预签名链接，客户端开发人员只需要完成 PUT HTTP 请求即可，不需要学习和调试不同语言下的 S3 SDK（事实证明，AWS SDK不同的语言版本由不同的团队维护，其规范、特性差异较大，不利于项目稳定。）
4. 高性能：上传请求依然是直接对 缤纷云 S4 发起，不需要服务端中转。

如何运作

1. 通过后端 API 生成用于上传的预签名链接：putObject-presigned-url

   让后端程序（golang、python、php、nodejs）调用相应的 SDK 生成一个具有 PutObject 方法效力的「预签名」链接传递给 Web 页面的 Javascript 脚本。

2. 客户端对 putObject-presigned-url 发起 PUT 请求，并完成文件上传

   Javascript 脚本利用 网络请求组件（如 axios）来将本地文件 PUT 到上面提到的「预签名 PutObject 链接」

预签名 URL API

缤纷云开源了全功能预签名 API 本地部署项目。它无状态，可以作为微服务快速结合到当前的系统中。

Github: https://github.com/bitiful/s3-presigned-api-server

不同语言后端接口生成 PutObject 预签名链接示例：

main.py

import boto3
from flask import Flask
from flask import render_template_string
from flask import request

app = Flask(__name__)

@app.get('/s3_upload_url')
def get_upload_url():
    # Config
    s3endpoint = 'https://s3.bitiful.net' # 请填入控制台 “Bucket 设置” 页面底部的 “Endpoint” 标签中的信息
    s3region = 'cn-east-1'
    s3accessKeyId = '<--子账户 accessKey-->' # 请到控制台创建子账户，并为子账户创建相应 accessKey
    s3SecretKeyId = '<--子账户 secretKey-->' # ！！切记，创建子账户时，需要手动为其分配具体权限！！

    # 连接 S3
    client = boto3.client(
        's3',
        aws_access_key_id = s3accessKeyId,
        aws_secret_access_key = s3SecretKeyId,
        endpoint_url = s3endpoint,
        region_name = s3region
    )
    url = client.generate_presigned_url(
        'put_object',
        Params={
            'Bucket': '<--缤纷云存储桶名-->',
            'Key': request.args.get('key'),
        },
        ExpiresIn=3600
    )
    return {'url': url}

// 参考 1：https://juejin.cn/post/7019216042220077092
// 参考 2：https://northflank.com/guides/connect-nodejs-to-minio-with-tls-using-aws-s3
const {
  S3Client
} = require("@aws-sdk/client-s3");
const s3Client = new S3Client({
  endpoint: "https://s3.bitiful.net",
  credentials: {
    accessKeyId: "<--子账户 accessKey-->",
    secretAccessKey: "<--子账户 secretKey-->",
  }
});

//引入相关模块
const {
  PutObjectCommand,
} = require("@aws-sdk/client-s3");
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");

app.get("/s3_upload_url", (req, res, next) => {
  //初始化命令实体
  const putCmd = new PutObjectCommand({
    Bucket: "<--缤纷云存储桶名-->",
    Key: req.query.key
  });
  //获取签名
  getSignedUrl(s3Client, putCmd, { expiresIn: 3600 }).then((url) => {
      //将签名好的url回传给前台
    res.send(url);
    next();
  });
});

// 参考：https://developer.qiniu.com/kodo/12572/aws-sdk-rust-examples
// [dependencies]
// actix-web = "4"
// serde = { version = "1.0", features = ["derive"] }
// serde_json = "1.0"
// anyhow = "1.0.75"
// aws-config = "1.3.0"
// aws-credential-types = { version = "1.2.0", features = ["hardcoded-credentials"] }
// aws-sdk-s3 = "1.25.0"
// aws-sdk-sts = "1.22.0"
// aws-smithy-types = { version = "1.1.8", features = ["rt-tokio"] }
// tokio = { version = "1.32.0", features = ["full"] }

use actix_web::{web, App, HttpResponse, HttpServer, Responder};
use aws_config::{BehaviorVersion, SdkConfig};
use aws_credential_types::{provider::SharedCredentialsProvider, Credentials};
use aws_sdk_s3::{config::Region, presigning::PresigningConfig, Client};
use std::time::Duration;

async fn get_upload_url(query: web::Query<serde_json::Value>) -> impl Responder {
    let key = query.get("key").and_then(|v| v.as_str()).unwrap_or("");
    let url = get_url(key);

    let response_data = serde_json::json!({ "url": url.await.unwrap_or("".to_string())});
    HttpResponse::Ok()
        .content_type("application/json") // 设置响应的 Content-Type
        .json(response_data)
}

#[actix_web::main]
async fn main() -> std::io::Result<()> {
    HttpServer::new(|| {
        App::new()
            .route("/s3_upload_url", web::get().to(get_upload_url))
    })
    .bind("127.0.0.1:8080")?
    .run()
    .await
}

async fn get_url(key: &str) -> anyhow::Result<String> {
    let client = Client::new(
        &SdkConfig::builder()
            .region(Region::new("cn-east-1"))
            .endpoint_url("https://s3.bitiful.net")
            .behavior_version(BehaviorVersion::v2023_11_09())
            .credentials_provider(SharedCredentialsProvider::new(Credentials::from_keys(
                "<--子账户 accessKey-->",
                "<--子账户 secretKey-->",
                None,
            )))
            .build(),
    );

    let request = client
        .put_object()
        .bucket("<--缤纷云存储桶名-->")
        .key(key)
        .presigned(
            PresigningConfig::builder()
                .expires_in(Duration::from_secs(36000))
        .build()?,
        )
        .await?;

    Ok(request.uri().to_string())
}

// go get github.com/aws/aws-sdk-go

package main

import (
    "encoding/json"
    "log"
    "net/http"
    "time"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/credentials"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

const (
    bucket     = "<--缤纷云存储桶名-->"
    endpoint   = "https://s3.bitiful.net"
    region     = "cn-east-1"
    accessKey  = "<--子账户 accessKey-->"
    secretKey  = "<--子账户 secretKey-->"
)

func main() {
    http.HandleFunc("/s3_upload_url", handleS3UploadURL)
    log.Println("Server started at http://localhost:8080")
    log.Fatal(http.ListenAndServe(":8080", nil))
}

func handleS3UploadURL(w http.ResponseWriter, r *http.Request) {
    // Parse query parameter
    keys, ok := r.URL.Query()["key"]
    if !ok || len(keys[0]) < 1 {
        http.Error(w, "Missing 'key' query parameter", http.StatusBadRequest)
        return
    }
    key := keys[0]

    // Create AWS session
    sess := session.Must(session.NewSession(&aws.Config{
        Region:      aws.String(region),
        Endpoint:    aws.String(endpoint),
        Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
    }))

    // Create S3 service client
    svc := s3.New(sess)

    // Generate the URL
    req, _ := svc.PutObjectRequest(&s3.PutObjectInput{
        Bucket: aws.String(bucket),
        Key:    aws.String(key),
    })
    urlStr, err := req.Presign(15 * time.Minute) // URL expires in 15 minutes
    if err != nil {
        http.Error(w, "Failed to sign request: "+err.Error(), http.StatusInternalServerError)
        return
    }

    // Create JSON response
    response := struct {
        URL string `json:"url"`
    }{
        URL: urlStr,
    }
    jsonResponse, err := json.Marshal(response)
    if err != nil {
        http.Error(w, "JSON encoding error: "+err.Error(), http.StatusInternalServerError)
        return
    }

    // Set Content-Type and write the response
    w.Header().Set("Content-Type", "application/json")
    w.Write(jsonResponse)
}

# composer require aws/aws-sdk-php

require 'vendor/autoload.php';

use Aws\S3\S3Client;
use Aws\Exception\AwsException;

// 配置
$bucket = '<--缤纷云存储桶名-->';
$region = 'cn-east-1';
$keyId = '<--子账户 accessKey-->';
$secret = '<--子账户 secretKey-->';
$endpoint = 'https://s3.bitiful.net';

// 创建 S3 客户端
$s3Client = new S3Client([
    'version'     => 'latest',
    'region'      => $region,
    'credentials' => [
        'key'    => $keyId,
        'secret' => $secret,
    ],
    'endpoint' => $endpoint,
    'use_path_style_endpoint' => true,
]);

// 检查是否有 key 参数
if (!isset($_GET['key']) || empty($_GET['key'])) {
    header('Content-Type: application/json');
    echo json_encode(['error' => 'Missing key parameter']);
    http_response_code(400);
    exit;
}

$key = $_GET['key'];

// 生成预签名 URL
try {
    $cmd = $s3Client->getCommand('PutObject', [
        'Bucket' => $bucket,
        'Key'    => $key
    ]);

    $request = $s3Client->createPresignedRequest($cmd, '+20 minutes');

    // 获取预签名 URL
    $presignedUrl = (string) $request->getUri();

    header('Content-Type: application/json');
    echo json_encode(['url' => $presignedUrl]);
} catch (AwsException $e) {
    header('Content-Type: application/json');
    echo json_encode(['error' => $e->getMessage()]);
    http_response_code(500);
}

前端上传示例：

upload_demo.web.html

<!DOCTYPE html>
<html>
<head>
  <title>S3 File Upload</title>
</head>
<body>
<div>
  <h1>S3 File Upload</h1>
  <input type="file" id="file">
  <button onclick="handleClick()">Upload</button>
</div>
<script src="https://cdn.bootcdn.net/ajax/libs/axios/1.3.6/axios.min.js"></script>
<script>
    let file;
    let fileContent;

    const fileInput = document.getElementById('file');
    const fileReader = new FileReader();

    fileInput.addEventListener('change', function (event) {
        file = event.target.files[0];
        console.log("file is: ", file);
        fileReader.readAsArrayBuffer(file);
    });

    fileReader.onload = function (event) {
        fileContent = event.target.result;
        // console.log("fileContent is: ", fileContent);
    };

    function handleClick() {
        console.log('click');

        const url = `/s3_upload_url`;
        const params = {key: '1.jpg'};

        axios.get(url, {params: params})
            .then(function (res) {
                console.log(res);
                const url = res.data.url;
                const config = {
                    headers: {
                        'Content-Type': null,
                    },
                };
                axios.put(url, file, config)
                    .then(function (res) {
                        console.log("res is: ", res);
                    });
            });

    }
</script>

</body>
</html>

上面的 Javascript 脚本在发起 Put 请求时将「Content-Type」设置为空值，建议使用缤纷云独有的 MIME 自动检测 功能。