blog/go/oss_image_resize/main.go

538 lines
12 KiB
Go
Raw Normal View History

2023-03-13 16:17:29 +08:00
package main
import (
"bytes"
"flag"
"fmt"
"io"
"net/http"
"os"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
)
const (
metaKeyUpdatedAt = "UpdatedAt"
metaKeyModifiedAt = "Last-Modified"
metaKeyFileSize = "Content-Length"
)
type AliOSS struct {
bucket *oss.Bucket
client *oss.Client
endpoint string
accessKey string
secret string
bucketName string
}
func NewAliOSS(endpoint, accessKey, secret string, bucket string) *AliOSS {
return &AliOSS{
endpoint: endpoint,
accessKey: accessKey,
secret: secret,
bucketName: bucket,
}
}
func (a *AliOSS) Init(opts ...oss.ClientOption) error {
client, err := oss.New(a.endpoint, a.accessKey, a.secret, opts...)
if err != nil {
return err
}
a.client = client
a.bucket, err = client.Bucket(a.bucketName)
return err
}
// Copy 拷贝文件
func (a *AliOSS) Copy(src, dst string, opts ...oss.Option) error {
_, err := a.bucket.CopyObject(src, dst, opts...)
return err
}
func (a *AliOSS) Delete(f []string, opts ...oss.Option) ([]string, error) {
res, err := a.bucket.DeleteObjects(f, opts...)
return res.DeletedObjects, err
}
// Download 下载文件
func (a *AliOSS) Download(obj, savePath string, opts ...oss.Option) error {
err := a.bucket.GetObjectToFile(obj, savePath, opts...)
return err
}
// IsExist 判断文件是否存在
func (a *AliOSS) IsExist(obj string) (bool, error) {
return a.bucket.IsObjectExist(obj)
}
func (a *AliOSS) UploadObject(dest string, src io.Reader, opts ...oss.Option) error {
return a.bucket.PutObject(dest, src, opts...)
}
func (a *AliOSS) UploadByte(dest string, buf []byte, opts ...oss.Option) error {
return a.UploadObject(dest, bytes.NewReader(buf), opts...)
}
func (a *AliOSS) UploadStream(dest string, src *os.File, opts ...oss.Option) error {
// 读取本地文件。
return a.UploadObject(dest, src, opts...)
}
func (a *AliOSS) UploadFile(dest string, filePath string, opts ...oss.Option) error {
err := a.bucket.PutObjectFromFile(dest, filePath, opts...)
return err
}
// set meta
func (a *AliOSS) SetObjectMeta(file string, key, val string) error {
meta := oss.Meta(key, val)
err := a.bucket.SetObjectMeta(file, meta)
return err
}
// set metas
func (a *AliOSS) SetObjectMetaMap(file string, metaMap map[string]string) error {
metas := []oss.Option{}
for k, v := range metaMap {
metas = append(metas, oss.Meta(k, v))
}
if len(metas) == 0 {
return nil
}
return a.bucket.SetObjectMeta(file, metas...)
}
// get metas
func (a *AliOSS) GetObjectMeta(file string) (map[string]string, error) {
res := make(map[string]string)
metas, err := a.bucket.GetObjectMeta(file)
for k, v := range metas {
val := ""
if len(v) > 0 {
val = v[0]
}
res[k] = val
}
return res, err
}
// set updatedAt
func (a *AliOSS) SetUpdatedAt(file string, updatedAt *time.Time) error {
if updatedAt == nil {
now := time.Now()
updatedAt = &now
}
return a.SetObjectMeta(file, metaKeyUpdatedAt, strconv.FormatInt(updatedAt.Unix(), 10))
}
// get updatedAt
func (a *AliOSS) GetUpdatedAt(file string) (int64, error) {
metas, err := a.GetObjectMeta(file)
if err != nil {
return 0, err
}
if u, ok := metas[metaKeyUpdatedAt]; ok {
i, err := strconv.ParseInt(u, 10, 64)
return i, err
}
return 0, nil
}
// get updatedAt
func (a *AliOSS) GetModifiedAt(file string) (int64, error) {
metas, err := a.GetObjectMeta(file)
if err != nil {
return 0, err
}
if u, ok := metas[metaKeyModifiedAt]; ok {
//Thu, 07 Apr 2022 03:13:49 GMT
t, err := time.Parse(time.RFC1123, u)
return t.Unix(), err
}
return 0, nil
}
// upload stream with set updated at
func (a *AliOSS) UploadObjectWithUpdatedAt(dest string, src io.Reader, opts ...oss.Option) error {
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
opts = append(opts, opt)
return a.UploadObject(dest, src, opts...)
}
// upload file with set updated at
func (a *AliOSS) UploadFileWithUpdatedAt(dest string, src string, opts ...oss.Option) error {
err := a.UploadFile(dest, src, opts...)
if err != nil {
return err
}
return a.SetUpdatedAt(dest, nil)
}
// upload bytes with set updated at
func (a *AliOSS) UploadBytesWithUpdatedAt(dest string, src []byte, opts ...oss.Option) error {
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
opts = append(opts, opt)
return a.UploadByte(dest, src, opts...)
}
func (a *AliOSS) Name() string {
return a.bucketName
}
// check file version, if not need download ,return true
func (a *AliOSS) CheckFileVersion(obj string, info os.FileInfo) (bool, error) {
metas, err := a.GetObjectMeta(obj)
if err != nil {
return true, err
}
// var modifiedAt time.Time
var modifiedAt int64
if u, ok := metas[metaKeyModifiedAt]; ok {
//Thu, 07 Apr 2022 03:13:49 GMT
t, err := time.Parse(time.RFC1123, u)
if err == nil {
modifiedAt = t.Unix()
}
// modifiedAt, _ = strconv.ParseInt(u, 10, 64)
}
//TODO is need to check file size??
// var length int64 = 0
// if u, ok := metas[metaKeyFileSize]; ok {
// n, err := strconv.ParseInt(u, 10, 64)
// if err == nil {
// length = n
// }
// }
//如果本地文件时间更靠后或者本地文件更大返回true
// return info.ModTime().Unix() >= modifiedAt || length <= info.Size(), nil
//如果本地文件时间更靠后返回true, 防止意外退出,未成功上传
return info.ModTime().Unix() >= modifiedAt, nil
}
// const host = "http://testspeed.xk.design"
const MAX = 300
type file struct{
Name string
Width int
Height int
Src string
}
var test1 = []file{
{
Name: "test4k.jpg",
Width: 0,
Height: 0,
Src: "test4k.jpg",
},
{
Name: "test2k.jpg",
Width: 2590,
Height: 1619,
Src: "test4k.jpg",
},
{
Name: "test1k.jpg",
Width: 1831,
Height: 1144,
Src: "test4k.jpg",
},
{
Name: "test600.jpg",
Width: 647,
Height: 404,
Src: "test4k.jpg",
},
{
Name: "test1of10.jpg",
Width: 384,
Height: 240,
Src: "test4k.jpg",
},
}
var test2 = []file{
{
Name: "8ktestsrc.jpeg",
Width: 7680,
Height: 4320,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest3k.jpeg",
Width: 2730,
Height: 1536,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest1080.jpeg",
Width: 1930,
Height: 1086,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest682.jpeg",
Width: 682,
Height: 384,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest400.jpeg",
Width: 400,
Height: 225,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest300.jpeg",
Width: 300,
Height: 300,
Src: "8ktestsrc.jpeg",
},
}
func NewRequest(url string)(*http.Request, error){
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Add("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
req.Header.Add("Accept-Encoding","gzip, deflate")
req.Header.Add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56")
req.Header.Add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
req.Header.Add("Cache-Control","no-cache")
req.Header.Add("Connection"," keep-alive")
req.Header.Add("Cookie"," Hm_lvt_2e2c80c1bd8a51afc2d7de641330a397=1669089775,1669615598,1669704782,1669875251; Hm_lpvt_2e2c80c1bd8a51afc2d7de641330a397=1669964998")
req.Header.Add("Host"," test-pic.xk.design")
// req.Header.Add("If-Modified-Since"," Fri, 02 Dec 2022 06:31:28 GMT")
// req.Header.Add("If-None-Match","\"4B8579BA5E07DD57405973606E777476\"")
// req.Header.Add("Upgrade-Insecure-Requests","1")
return req, nil
}
func test(f file, count int, host string){
fmt.Printf("begin normal testing for %s\n", f.Name)
startedAt := time.Now().UnixMilli()
errCount := 0
costMS := []int64{}
for i := 0; i < count; i ++ {
req, err := NewRequest(host + "/" + f.Name)
if err != nil {
errCount ++
continue
}
client := &http.Client{}
begin := time.Now().UnixMilli()
resp, err := client.Do(req)
if err != nil {
errCount++
continue
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
os.WriteFile("/dev/null", data, os.ModePerm)
if err != nil {
errCount++
continue
}
end := time.Now().UnixMilli()
costMS = append(costMS, end-begin)
}
finishedAt := time.Now().UnixMilli()
totalCost := finishedAt - startedAt
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
statistics(costMS)
}
func testResize(f file, count int, host string)(int64, int64, int64){
fmt.Printf("begin resize testing for %s\n", f.Name)
startedAt := time.Now().UnixMilli()
errCount := 0
costMS := []int64{}
for i := 0; i < count; i ++ {
query := "?x-oss-process=image/resize,m_lfit,w_"+strconv.Itoa(f.Width) + ",h_" + strconv.Itoa(f.Height)
if f.Height == 0 || f.Width == 0 {
query = ""
}
req, err := NewRequest(host + "/" + f.Src + query)
if err != nil {
errCount++
continue
}
client := http.Client{}
begin := time.Now().UnixMilli()
resp, err := client.Do(req)
if err != nil {
errCount++
continue
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
os.WriteFile("/dev/null", data, os.ModePerm)
if err != nil {
errCount++
continue
}
end := time.Now().UnixMilli()
costMS = append(costMS, end-begin)
// if resp != nil {
// fmt.Println(resp , resp.StatusCode, resp.Status)
// }
}
finishedAt := time.Now().UnixMilli()
totalCost := finishedAt - startedAt
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
return statistics(costMS)
}
func filelist(dir string)([]string, error){
fis, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
fs := []string{}
for _, f := range fis {
if !f.IsDir() {
fs = append(fs, f.Name())
}
}
return fs, nil
}
func case3(dir string, ak string, sk string, count int, host string)error{
files, err := filelist(dir)
if err != nil {
return err
}
alioss := NewAliOSS("oss-cn-hangzhou.aliyuncs.com", ak,sk, "oxslmimg")
alioss.Init()
avgmap := make(map[string][]string)
p95map := make(map[string][]string)
for _, f := range files {
err = alioss.UploadFile(f, path.Join(dir,f))
if err != nil {
fmt.Println("upload failed:", err, f, path.Join(dir,f))
continue
}
avgmap[f] = []string{}
p95map[f] = []string{}
for _, t := range test1 {
t.Src = f
t.Name = f
avg, p95, _ := testResize(t, count, host)
avgmap[f] = append(avgmap[f], strconv.FormatInt(avg, 10))
p95map[f] = append(p95map[f], strconv.FormatInt(p95, 10))
}
}
fmt.Println("")
for f, v := range avgmap {
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
}
for f, v := range p95map {
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
}
return nil
}
func avg(data []int64)int64{
if len(data) == 0 {
return 0
}
total := int64(0)
for _, i := range data {
total = total + i
}
return total/int64(len(data))
}
type SortInt64 []int64
func (a SortInt64) Len() int { return len(a) }
func (a SortInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a SortInt64) Less(i, j int) bool { return a[i] < a[j] }
func percent50(data []int64)int64{
if len(data) == 0 {
return 0
}
i := int(float64(len(data)) * 0.5 )
if i == len(data) {
return 0
}
return data[i]
}
func percent90(data []int64)int64{
if len(data) == 0 {
return 0
}
i := int(float64(len(data)) * 0.9 )
if i == len(data) {
return 0
}
return data[i]
}
func percent95(data []int64)int64{
if len(data) == 0 {
return 0
}
i := int(float64(len(data)) * 0.95 )
if i == len(data) {
return 0
}
return data[i]
}
func max(data []int64)int64{
return data[len(data)-1]
}
func statistics(data []int64)(int64, int64, int64){
a := avg(data)
sort.Sort(SortInt64(data))
m := percent50(data)
p9 := percent90(data)
p95 := percent95(data)
ma := max(data)
fmt.Printf("\t avg: %d middle: %dms percent90: %dms percent95: %dms max: %dms\n", a, m, p9, p95, ma)
return a, p95, ma
}
func main(){
var tcase, count int
var host, ak, sk, dir string
flag.IntVar(&tcase, "case", 1, "which test case to use")
flag.IntVar(&count, "count", 100, "total test count")
flag.StringVar(&host, "host", "http://testspeed.xk.design", "access host")
flag.StringVar(&ak, "ak", "", "access key")
flag.StringVar(&sk, "sk", "", "access secret")
flag.StringVar(&dir , "dir", "", "img path")
flag.Parse()
switch tcase {
case 1:
for _, f := range test1 {
test(f, count, host)
testResize(f, count, host)
}
case 2:
for _, f := range test2 {
test(f, count, host)
testResize(f, count, host)
}
case 3:
case3(dir, ak, sk, count, host)
}
}