package main import ( "bytes" "flag" "fmt" "io" "net/http" "os" "path" "sort" "strconv" "strings" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) const ( metaKeyUpdatedAt = "UpdatedAt" metaKeyModifiedAt = "Last-Modified" metaKeyFileSize = "Content-Length" ) type AliOSS struct { bucket *oss.Bucket client *oss.Client endpoint string accessKey string secret string bucketName string } func NewAliOSS(endpoint, accessKey, secret string, bucket string) *AliOSS { return &AliOSS{ endpoint: endpoint, accessKey: accessKey, secret: secret, bucketName: bucket, } } func (a *AliOSS) Init(opts ...oss.ClientOption) error { client, err := oss.New(a.endpoint, a.accessKey, a.secret, opts...) if err != nil { return err } a.client = client a.bucket, err = client.Bucket(a.bucketName) return err } // Copy 拷贝文件 func (a *AliOSS) Copy(src, dst string, opts ...oss.Option) error { _, err := a.bucket.CopyObject(src, dst, opts...) return err } func (a *AliOSS) Delete(f []string, opts ...oss.Option) ([]string, error) { res, err := a.bucket.DeleteObjects(f, opts...) return res.DeletedObjects, err } // Download 下载文件 func (a *AliOSS) Download(obj, savePath string, opts ...oss.Option) error { err := a.bucket.GetObjectToFile(obj, savePath, opts...) return err } // IsExist 判断文件是否存在 func (a *AliOSS) IsExist(obj string) (bool, error) { return a.bucket.IsObjectExist(obj) } func (a *AliOSS) UploadObject(dest string, src io.Reader, opts ...oss.Option) error { return a.bucket.PutObject(dest, src, opts...) } func (a *AliOSS) UploadByte(dest string, buf []byte, opts ...oss.Option) error { return a.UploadObject(dest, bytes.NewReader(buf), opts...) } func (a *AliOSS) UploadStream(dest string, src *os.File, opts ...oss.Option) error { // 读取本地文件。 return a.UploadObject(dest, src, opts...) } func (a *AliOSS) UploadFile(dest string, filePath string, opts ...oss.Option) error { err := a.bucket.PutObjectFromFile(dest, filePath, opts...) return err } // set meta func (a *AliOSS) SetObjectMeta(file string, key, val string) error { meta := oss.Meta(key, val) err := a.bucket.SetObjectMeta(file, meta) return err } // set metas func (a *AliOSS) SetObjectMetaMap(file string, metaMap map[string]string) error { metas := []oss.Option{} for k, v := range metaMap { metas = append(metas, oss.Meta(k, v)) } if len(metas) == 0 { return nil } return a.bucket.SetObjectMeta(file, metas...) } // get metas func (a *AliOSS) GetObjectMeta(file string) (map[string]string, error) { res := make(map[string]string) metas, err := a.bucket.GetObjectMeta(file) for k, v := range metas { val := "" if len(v) > 0 { val = v[0] } res[k] = val } return res, err } // set updatedAt func (a *AliOSS) SetUpdatedAt(file string, updatedAt *time.Time) error { if updatedAt == nil { now := time.Now() updatedAt = &now } return a.SetObjectMeta(file, metaKeyUpdatedAt, strconv.FormatInt(updatedAt.Unix(), 10)) } // get updatedAt func (a *AliOSS) GetUpdatedAt(file string) (int64, error) { metas, err := a.GetObjectMeta(file) if err != nil { return 0, err } if u, ok := metas[metaKeyUpdatedAt]; ok { i, err := strconv.ParseInt(u, 10, 64) return i, err } return 0, nil } // get updatedAt func (a *AliOSS) GetModifiedAt(file string) (int64, error) { metas, err := a.GetObjectMeta(file) if err != nil { return 0, err } if u, ok := metas[metaKeyModifiedAt]; ok { //Thu, 07 Apr 2022 03:13:49 GMT t, err := time.Parse(time.RFC1123, u) return t.Unix(), err } return 0, nil } // upload stream with set updated at func (a *AliOSS) UploadObjectWithUpdatedAt(dest string, src io.Reader, opts ...oss.Option) error { opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10)) opts = append(opts, opt) return a.UploadObject(dest, src, opts...) } // upload file with set updated at func (a *AliOSS) UploadFileWithUpdatedAt(dest string, src string, opts ...oss.Option) error { err := a.UploadFile(dest, src, opts...) if err != nil { return err } return a.SetUpdatedAt(dest, nil) } // upload bytes with set updated at func (a *AliOSS) UploadBytesWithUpdatedAt(dest string, src []byte, opts ...oss.Option) error { opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10)) opts = append(opts, opt) return a.UploadByte(dest, src, opts...) } func (a *AliOSS) Name() string { return a.bucketName } // check file version, if not need download ,return true func (a *AliOSS) CheckFileVersion(obj string, info os.FileInfo) (bool, error) { metas, err := a.GetObjectMeta(obj) if err != nil { return true, err } // var modifiedAt time.Time var modifiedAt int64 if u, ok := metas[metaKeyModifiedAt]; ok { //Thu, 07 Apr 2022 03:13:49 GMT t, err := time.Parse(time.RFC1123, u) if err == nil { modifiedAt = t.Unix() } // modifiedAt, _ = strconv.ParseInt(u, 10, 64) } //TODO is need to check file size?? // var length int64 = 0 // if u, ok := metas[metaKeyFileSize]; ok { // n, err := strconv.ParseInt(u, 10, 64) // if err == nil { // length = n // } // } //如果本地文件时间更靠后或者本地文件更大,返回true // return info.ModTime().Unix() >= modifiedAt || length <= info.Size(), nil //如果本地文件时间更靠后,返回true, 防止意外退出,未成功上传 return info.ModTime().Unix() >= modifiedAt, nil } // const host = "http://testspeed.xk.design" const MAX = 300 type file struct{ Name string Width int Height int Src string } var test1 = []file{ { Name: "test4k.jpg", Width: 0, Height: 0, Src: "test4k.jpg", }, { Name: "test2k.jpg", Width: 2590, Height: 1619, Src: "test4k.jpg", }, { Name: "test1k.jpg", Width: 1831, Height: 1144, Src: "test4k.jpg", }, { Name: "test600.jpg", Width: 647, Height: 404, Src: "test4k.jpg", }, { Name: "test1of10.jpg", Width: 384, Height: 240, Src: "test4k.jpg", }, } var test2 = []file{ { Name: "8ktestsrc.jpeg", Width: 7680, Height: 4320, Src: "8ktestsrc.jpeg", }, { Name: "8ktest3k.jpeg", Width: 2730, Height: 1536, Src: "8ktestsrc.jpeg", }, { Name: "8ktest1080.jpeg", Width: 1930, Height: 1086, Src: "8ktestsrc.jpeg", }, { Name: "8ktest682.jpeg", Width: 682, Height: 384, Src: "8ktestsrc.jpeg", }, { Name: "8ktest400.jpeg", Width: 400, Height: 225, Src: "8ktestsrc.jpeg", }, { Name: "8ktest300.jpeg", Width: 300, Height: 300, Src: "8ktestsrc.jpeg", }, } func NewRequest(url string)(*http.Request, error){ req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, err } req.Header.Add("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") req.Header.Add("Accept-Encoding","gzip, deflate") req.Header.Add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56") req.Header.Add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") req.Header.Add("Cache-Control","no-cache") req.Header.Add("Connection"," keep-alive") req.Header.Add("Cookie"," Hm_lvt_2e2c80c1bd8a51afc2d7de641330a397=1669089775,1669615598,1669704782,1669875251; Hm_lpvt_2e2c80c1bd8a51afc2d7de641330a397=1669964998") req.Header.Add("Host"," test-pic.xk.design") // req.Header.Add("If-Modified-Since"," Fri, 02 Dec 2022 06:31:28 GMT") // req.Header.Add("If-None-Match","\"4B8579BA5E07DD57405973606E777476\"") // req.Header.Add("Upgrade-Insecure-Requests","1") return req, nil } func test(f file, count int, host string){ fmt.Printf("begin normal testing for %s\n", f.Name) startedAt := time.Now().UnixMilli() errCount := 0 costMS := []int64{} for i := 0; i < count; i ++ { req, err := NewRequest(host + "/" + f.Name) if err != nil { errCount ++ continue } client := &http.Client{} begin := time.Now().UnixMilli() resp, err := client.Do(req) if err != nil { errCount++ continue } defer resp.Body.Close() data, err := io.ReadAll(resp.Body) os.WriteFile("/dev/null", data, os.ModePerm) if err != nil { errCount++ continue } end := time.Now().UnixMilli() costMS = append(costMS, end-begin) } finishedAt := time.Now().UnixMilli() totalCost := finishedAt - startedAt fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount) statistics(costMS) } func testResize(f file, count int, host string)(int64, int64, int64){ fmt.Printf("begin resize testing for %s\n", f.Name) startedAt := time.Now().UnixMilli() errCount := 0 costMS := []int64{} for i := 0; i < count; i ++ { query := "?x-oss-process=image/resize,m_lfit,w_"+strconv.Itoa(f.Width) + ",h_" + strconv.Itoa(f.Height) if f.Height == 0 || f.Width == 0 { query = "" } req, err := NewRequest(host + "/" + f.Src + query) if err != nil { errCount++ continue } client := http.Client{} begin := time.Now().UnixMilli() resp, err := client.Do(req) if err != nil { errCount++ continue } defer resp.Body.Close() data, err := io.ReadAll(resp.Body) os.WriteFile("/dev/null", data, os.ModePerm) if err != nil { errCount++ continue } end := time.Now().UnixMilli() costMS = append(costMS, end-begin) // if resp != nil { // fmt.Println(resp , resp.StatusCode, resp.Status) // } } finishedAt := time.Now().UnixMilli() totalCost := finishedAt - startedAt fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount) return statistics(costMS) } func filelist(dir string)([]string, error){ fis, err := os.ReadDir(dir) if err != nil { return nil, err } fs := []string{} for _, f := range fis { if !f.IsDir() { fs = append(fs, f.Name()) } } return fs, nil } func case3(dir string, ak string, sk string, count int, host string)error{ files, err := filelist(dir) if err != nil { return err } alioss := NewAliOSS("oss-cn-hangzhou.aliyuncs.com", ak,sk, "oxslmimg") alioss.Init() avgmap := make(map[string][]string) p95map := make(map[string][]string) for _, f := range files { err = alioss.UploadFile(f, path.Join(dir,f)) if err != nil { fmt.Println("upload failed:", err, f, path.Join(dir,f)) continue } avgmap[f] = []string{} p95map[f] = []string{} for _, t := range test1 { t.Src = f t.Name = f avg, p95, _ := testResize(t, count, host) avgmap[f] = append(avgmap[f], strconv.FormatInt(avg, 10)) p95map[f] = append(p95map[f], strconv.FormatInt(p95, 10)) } } fmt.Println("") for f, v := range avgmap { fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t")) } for f, v := range p95map { fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t")) } return nil } func avg(data []int64)int64{ if len(data) == 0 { return 0 } total := int64(0) for _, i := range data { total = total + i } return total/int64(len(data)) } type SortInt64 []int64 func (a SortInt64) Len() int { return len(a) } func (a SortInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a SortInt64) Less(i, j int) bool { return a[i] < a[j] } func percent50(data []int64)int64{ if len(data) == 0 { return 0 } i := int(float64(len(data)) * 0.5 ) if i == len(data) { return 0 } return data[i] } func percent90(data []int64)int64{ if len(data) == 0 { return 0 } i := int(float64(len(data)) * 0.9 ) if i == len(data) { return 0 } return data[i] } func percent95(data []int64)int64{ if len(data) == 0 { return 0 } i := int(float64(len(data)) * 0.95 ) if i == len(data) { return 0 } return data[i] } func max(data []int64)int64{ return data[len(data)-1] } func statistics(data []int64)(int64, int64, int64){ a := avg(data) sort.Sort(SortInt64(data)) m := percent50(data) p9 := percent90(data) p95 := percent95(data) ma := max(data) fmt.Printf("\t avg: %d middle: %dms percent90: %dms percent95: %dms max: %dms\n", a, m, p9, p95, ma) return a, p95, ma } func main(){ var tcase, count int var host, ak, sk, dir string flag.IntVar(&tcase, "case", 1, "which test case to use") flag.IntVar(&count, "count", 100, "total test count") flag.StringVar(&host, "host", "http://testspeed.xk.design", "access host") flag.StringVar(&ak, "ak", "", "access key") flag.StringVar(&sk, "sk", "", "access secret") flag.StringVar(&dir , "dir", "", "img path") flag.Parse() switch tcase { case 1: for _, f := range test1 { test(f, count, host) testResize(f, count, host) } case 2: for _, f := range test2 { test(f, count, host) testResize(f, count, host) } case 3: case3(dir, ak, sk, count, host) } }