bing
2 years ago
5 changed files with 620 additions and 0 deletions
@ -0,0 +1,538 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"bytes" |
|||
"flag" |
|||
"fmt" |
|||
"io" |
|||
"net/http" |
|||
"os" |
|||
"path" |
|||
"sort" |
|||
"strconv" |
|||
"strings" |
|||
"time" |
|||
|
|||
"github.com/aliyun/aliyun-oss-go-sdk/oss" |
|||
) |
|||
|
|||
const ( |
|||
metaKeyUpdatedAt = "UpdatedAt" |
|||
metaKeyModifiedAt = "Last-Modified" |
|||
metaKeyFileSize = "Content-Length" |
|||
) |
|||
|
|||
type AliOSS struct { |
|||
bucket *oss.Bucket |
|||
client *oss.Client |
|||
endpoint string |
|||
accessKey string |
|||
secret string |
|||
bucketName string |
|||
} |
|||
|
|||
func NewAliOSS(endpoint, accessKey, secret string, bucket string) *AliOSS { |
|||
return &AliOSS{ |
|||
endpoint: endpoint, |
|||
accessKey: accessKey, |
|||
secret: secret, |
|||
bucketName: bucket, |
|||
} |
|||
} |
|||
|
|||
func (a *AliOSS) Init(opts ...oss.ClientOption) error { |
|||
client, err := oss.New(a.endpoint, a.accessKey, a.secret, opts...) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
a.client = client |
|||
a.bucket, err = client.Bucket(a.bucketName) |
|||
return err |
|||
} |
|||
|
|||
// Copy 拷贝文件
|
|||
func (a *AliOSS) Copy(src, dst string, opts ...oss.Option) error { |
|||
|
|||
_, err := a.bucket.CopyObject(src, dst, opts...) |
|||
return err |
|||
} |
|||
|
|||
func (a *AliOSS) Delete(f []string, opts ...oss.Option) ([]string, error) { |
|||
res, err := a.bucket.DeleteObjects(f, opts...) |
|||
return res.DeletedObjects, err |
|||
} |
|||
|
|||
// Download 下载文件
|
|||
func (a *AliOSS) Download(obj, savePath string, opts ...oss.Option) error { |
|||
err := a.bucket.GetObjectToFile(obj, savePath, opts...) |
|||
return err |
|||
} |
|||
|
|||
// IsExist 判断文件是否存在
|
|||
func (a *AliOSS) IsExist(obj string) (bool, error) { |
|||
return a.bucket.IsObjectExist(obj) |
|||
} |
|||
|
|||
func (a *AliOSS) UploadObject(dest string, src io.Reader, opts ...oss.Option) error { |
|||
return a.bucket.PutObject(dest, src, opts...) |
|||
} |
|||
|
|||
func (a *AliOSS) UploadByte(dest string, buf []byte, opts ...oss.Option) error { |
|||
return a.UploadObject(dest, bytes.NewReader(buf), opts...) |
|||
} |
|||
|
|||
func (a *AliOSS) UploadStream(dest string, src *os.File, opts ...oss.Option) error { |
|||
// 读取本地文件。
|
|||
return a.UploadObject(dest, src, opts...) |
|||
} |
|||
|
|||
func (a *AliOSS) UploadFile(dest string, filePath string, opts ...oss.Option) error { |
|||
err := a.bucket.PutObjectFromFile(dest, filePath, opts...) |
|||
return err |
|||
} |
|||
|
|||
// set meta
|
|||
func (a *AliOSS) SetObjectMeta(file string, key, val string) error { |
|||
meta := oss.Meta(key, val) |
|||
err := a.bucket.SetObjectMeta(file, meta) |
|||
return err |
|||
} |
|||
|
|||
// set metas
|
|||
func (a *AliOSS) SetObjectMetaMap(file string, metaMap map[string]string) error { |
|||
metas := []oss.Option{} |
|||
for k, v := range metaMap { |
|||
metas = append(metas, oss.Meta(k, v)) |
|||
} |
|||
if len(metas) == 0 { |
|||
return nil |
|||
} |
|||
return a.bucket.SetObjectMeta(file, metas...) |
|||
} |
|||
|
|||
// get metas
|
|||
func (a *AliOSS) GetObjectMeta(file string) (map[string]string, error) { |
|||
res := make(map[string]string) |
|||
metas, err := a.bucket.GetObjectMeta(file) |
|||
for k, v := range metas { |
|||
val := "" |
|||
if len(v) > 0 { |
|||
val = v[0] |
|||
} |
|||
res[k] = val |
|||
} |
|||
return res, err |
|||
} |
|||
|
|||
// set updatedAt
|
|||
func (a *AliOSS) SetUpdatedAt(file string, updatedAt *time.Time) error { |
|||
if updatedAt == nil { |
|||
now := time.Now() |
|||
updatedAt = &now |
|||
} |
|||
return a.SetObjectMeta(file, metaKeyUpdatedAt, strconv.FormatInt(updatedAt.Unix(), 10)) |
|||
} |
|||
|
|||
// get updatedAt
|
|||
func (a *AliOSS) GetUpdatedAt(file string) (int64, error) { |
|||
metas, err := a.GetObjectMeta(file) |
|||
if err != nil { |
|||
return 0, err |
|||
} |
|||
if u, ok := metas[metaKeyUpdatedAt]; ok { |
|||
i, err := strconv.ParseInt(u, 10, 64) |
|||
return i, err |
|||
} |
|||
return 0, nil |
|||
} |
|||
|
|||
// get updatedAt
|
|||
func (a *AliOSS) GetModifiedAt(file string) (int64, error) { |
|||
metas, err := a.GetObjectMeta(file) |
|||
if err != nil { |
|||
return 0, err |
|||
} |
|||
if u, ok := metas[metaKeyModifiedAt]; ok { |
|||
//Thu, 07 Apr 2022 03:13:49 GMT
|
|||
t, err := time.Parse(time.RFC1123, u) |
|||
return t.Unix(), err |
|||
} |
|||
return 0, nil |
|||
} |
|||
|
|||
// upload stream with set updated at
|
|||
func (a *AliOSS) UploadObjectWithUpdatedAt(dest string, src io.Reader, opts ...oss.Option) error { |
|||
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10)) |
|||
opts = append(opts, opt) |
|||
return a.UploadObject(dest, src, opts...) |
|||
} |
|||
|
|||
// upload file with set updated at
|
|||
func (a *AliOSS) UploadFileWithUpdatedAt(dest string, src string, opts ...oss.Option) error { |
|||
err := a.UploadFile(dest, src, opts...) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
return a.SetUpdatedAt(dest, nil) |
|||
} |
|||
|
|||
// upload bytes with set updated at
|
|||
func (a *AliOSS) UploadBytesWithUpdatedAt(dest string, src []byte, opts ...oss.Option) error { |
|||
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10)) |
|||
opts = append(opts, opt) |
|||
return a.UploadByte(dest, src, opts...) |
|||
} |
|||
|
|||
func (a *AliOSS) Name() string { |
|||
return a.bucketName |
|||
} |
|||
|
|||
// check file version, if not need download ,return true
|
|||
func (a *AliOSS) CheckFileVersion(obj string, info os.FileInfo) (bool, error) { |
|||
metas, err := a.GetObjectMeta(obj) |
|||
if err != nil { |
|||
return true, err |
|||
} |
|||
// var modifiedAt time.Time
|
|||
var modifiedAt int64 |
|||
if u, ok := metas[metaKeyModifiedAt]; ok { |
|||
//Thu, 07 Apr 2022 03:13:49 GMT
|
|||
t, err := time.Parse(time.RFC1123, u) |
|||
if err == nil { |
|||
modifiedAt = t.Unix() |
|||
} |
|||
// modifiedAt, _ = strconv.ParseInt(u, 10, 64)
|
|||
} |
|||
//TODO is need to check file size??
|
|||
// var length int64 = 0
|
|||
// if u, ok := metas[metaKeyFileSize]; ok {
|
|||
// n, err := strconv.ParseInt(u, 10, 64)
|
|||
// if err == nil {
|
|||
// length = n
|
|||
// }
|
|||
// }
|
|||
//如果本地文件时间更靠后或者本地文件更大,返回true
|
|||
// return info.ModTime().Unix() >= modifiedAt || length <= info.Size(), nil
|
|||
//如果本地文件时间更靠后,返回true, 防止意外退出,未成功上传
|
|||
return info.ModTime().Unix() >= modifiedAt, nil |
|||
} |
|||
|
|||
|
|||
// const host = "http://testspeed.xk.design"
|
|||
|
|||
const MAX = 300 |
|||
|
|||
type file struct{ |
|||
Name string |
|||
Width int |
|||
Height int |
|||
Src string |
|||
} |
|||
|
|||
var test1 = []file{ |
|||
{ |
|||
Name: "test4k.jpg", |
|||
Width: 0, |
|||
Height: 0, |
|||
Src: "test4k.jpg", |
|||
}, |
|||
{ |
|||
Name: "test2k.jpg", |
|||
Width: 2590, |
|||
Height: 1619, |
|||
Src: "test4k.jpg", |
|||
}, |
|||
{ |
|||
Name: "test1k.jpg", |
|||
Width: 1831, |
|||
Height: 1144, |
|||
Src: "test4k.jpg", |
|||
}, |
|||
{ |
|||
Name: "test600.jpg", |
|||
Width: 647, |
|||
Height: 404, |
|||
Src: "test4k.jpg", |
|||
}, |
|||
{ |
|||
Name: "test1of10.jpg", |
|||
Width: 384, |
|||
Height: 240, |
|||
Src: "test4k.jpg", |
|||
}, |
|||
} |
|||
|
|||
var test2 = []file{ |
|||
{ |
|||
Name: "8ktestsrc.jpeg", |
|||
Width: 7680, |
|||
Height: 4320, |
|||
Src: "8ktestsrc.jpeg", |
|||
}, |
|||
{ |
|||
Name: "8ktest3k.jpeg", |
|||
Width: 2730, |
|||
Height: 1536, |
|||
Src: "8ktestsrc.jpeg", |
|||
}, |
|||
{ |
|||
Name: "8ktest1080.jpeg", |
|||
Width: 1930, |
|||
Height: 1086, |
|||
Src: "8ktestsrc.jpeg", |
|||
}, |
|||
{ |
|||
Name: "8ktest682.jpeg", |
|||
Width: 682, |
|||
Height: 384, |
|||
Src: "8ktestsrc.jpeg", |
|||
}, |
|||
{ |
|||
Name: "8ktest400.jpeg", |
|||
Width: 400, |
|||
Height: 225, |
|||
Src: "8ktestsrc.jpeg", |
|||
}, |
|||
{ |
|||
Name: "8ktest300.jpeg", |
|||
Width: 300, |
|||
Height: 300, |
|||
Src: "8ktestsrc.jpeg", |
|||
}, |
|||
} |
|||
|
|||
func NewRequest(url string)(*http.Request, error){ |
|||
req, err := http.NewRequest(http.MethodGet, url, nil) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
req.Header.Add("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") |
|||
req.Header.Add("Accept-Encoding","gzip, deflate") |
|||
req.Header.Add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56") |
|||
req.Header.Add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") |
|||
req.Header.Add("Cache-Control","no-cache") |
|||
req.Header.Add("Connection"," keep-alive") |
|||
req.Header.Add("Cookie"," Hm_lvt_2e2c80c1bd8a51afc2d7de641330a397=1669089775,1669615598,1669704782,1669875251; Hm_lpvt_2e2c80c1bd8a51afc2d7de641330a397=1669964998") |
|||
req.Header.Add("Host"," test-pic.xk.design") |
|||
// req.Header.Add("If-Modified-Since"," Fri, 02 Dec 2022 06:31:28 GMT")
|
|||
// req.Header.Add("If-None-Match","\"4B8579BA5E07DD57405973606E777476\"")
|
|||
// req.Header.Add("Upgrade-Insecure-Requests","1")
|
|||
return req, nil |
|||
} |
|||
|
|||
|
|||
func test(f file, count int, host string){ |
|||
fmt.Printf("begin normal testing for %s\n", f.Name) |
|||
startedAt := time.Now().UnixMilli() |
|||
errCount := 0 |
|||
costMS := []int64{} |
|||
for i := 0; i < count; i ++ { |
|||
req, err := NewRequest(host + "/" + f.Name) |
|||
if err != nil { |
|||
errCount ++ |
|||
continue |
|||
} |
|||
client := &http.Client{} |
|||
begin := time.Now().UnixMilli() |
|||
resp, err := client.Do(req) |
|||
if err != nil { |
|||
errCount++ |
|||
continue |
|||
} |
|||
defer resp.Body.Close() |
|||
data, err := io.ReadAll(resp.Body) |
|||
os.WriteFile("/dev/null", data, os.ModePerm) |
|||
if err != nil { |
|||
errCount++ |
|||
continue |
|||
} |
|||
end := time.Now().UnixMilli() |
|||
costMS = append(costMS, end-begin) |
|||
} |
|||
finishedAt := time.Now().UnixMilli() |
|||
totalCost := finishedAt - startedAt |
|||
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount) |
|||
statistics(costMS) |
|||
} |
|||
|
|||
func testResize(f file, count int, host string)(int64, int64, int64){ |
|||
fmt.Printf("begin resize testing for %s\n", f.Name) |
|||
startedAt := time.Now().UnixMilli() |
|||
errCount := 0 |
|||
costMS := []int64{} |
|||
for i := 0; i < count; i ++ { |
|||
query := "?x-oss-process=image/resize,m_lfit,w_"+strconv.Itoa(f.Width) + ",h_" + strconv.Itoa(f.Height) |
|||
if f.Height == 0 || f.Width == 0 { |
|||
query = "" |
|||
} |
|||
req, err := NewRequest(host + "/" + f.Src + query) |
|||
if err != nil { |
|||
errCount++ |
|||
continue |
|||
} |
|||
client := http.Client{} |
|||
begin := time.Now().UnixMilli() |
|||
resp, err := client.Do(req) |
|||
if err != nil { |
|||
errCount++ |
|||
continue |
|||
} |
|||
defer resp.Body.Close() |
|||
data, err := io.ReadAll(resp.Body) |
|||
os.WriteFile("/dev/null", data, os.ModePerm) |
|||
if err != nil { |
|||
errCount++ |
|||
continue |
|||
} |
|||
end := time.Now().UnixMilli() |
|||
costMS = append(costMS, end-begin) |
|||
// if resp != nil {
|
|||
// fmt.Println(resp , resp.StatusCode, resp.Status)
|
|||
// }
|
|||
} |
|||
finishedAt := time.Now().UnixMilli() |
|||
totalCost := finishedAt - startedAt |
|||
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount) |
|||
return statistics(costMS) |
|||
} |
|||
|
|||
func filelist(dir string)([]string, error){ |
|||
fis, err := os.ReadDir(dir) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
fs := []string{} |
|||
for _, f := range fis { |
|||
if !f.IsDir() { |
|||
fs = append(fs, f.Name()) |
|||
} |
|||
} |
|||
return fs, nil |
|||
} |
|||
|
|||
|
|||
func case3(dir string, ak string, sk string, count int, host string)error{ |
|||
files, err := filelist(dir) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
alioss := NewAliOSS("oss-cn-hangzhou.aliyuncs.com", ak,sk, "oxslmimg") |
|||
alioss.Init() |
|||
avgmap := make(map[string][]string) |
|||
p95map := make(map[string][]string) |
|||
for _, f := range files { |
|||
err = alioss.UploadFile(f, path.Join(dir,f)) |
|||
if err != nil { |
|||
fmt.Println("upload failed:", err, f, path.Join(dir,f)) |
|||
continue |
|||
} |
|||
avgmap[f] = []string{} |
|||
p95map[f] = []string{} |
|||
for _, t := range test1 { |
|||
t.Src = f |
|||
t.Name = f |
|||
avg, p95, _ := testResize(t, count, host) |
|||
avgmap[f] = append(avgmap[f], strconv.FormatInt(avg, 10)) |
|||
p95map[f] = append(p95map[f], strconv.FormatInt(p95, 10)) |
|||
} |
|||
} |
|||
fmt.Println("") |
|||
for f, v := range avgmap { |
|||
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t")) |
|||
} |
|||
for f, v := range p95map { |
|||
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t")) |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func avg(data []int64)int64{ |
|||
if len(data) == 0 { |
|||
return 0 |
|||
} |
|||
total := int64(0) |
|||
for _, i := range data { |
|||
total = total + i |
|||
} |
|||
return total/int64(len(data)) |
|||
} |
|||
|
|||
type SortInt64 []int64 |
|||
|
|||
func (a SortInt64) Len() int { return len(a) } |
|||
func (a SortInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] } |
|||
func (a SortInt64) Less(i, j int) bool { return a[i] < a[j] } |
|||
|
|||
func percent50(data []int64)int64{ |
|||
if len(data) == 0 { |
|||
return 0 |
|||
} |
|||
i := int(float64(len(data)) * 0.5 ) |
|||
if i == len(data) { |
|||
return 0 |
|||
} |
|||
return data[i] |
|||
} |
|||
|
|||
func percent90(data []int64)int64{ |
|||
if len(data) == 0 { |
|||
return 0 |
|||
} |
|||
i := int(float64(len(data)) * 0.9 ) |
|||
if i == len(data) { |
|||
return 0 |
|||
} |
|||
return data[i] |
|||
} |
|||
|
|||
func percent95(data []int64)int64{ |
|||
if len(data) == 0 { |
|||
return 0 |
|||
} |
|||
i := int(float64(len(data)) * 0.95 ) |
|||
if i == len(data) { |
|||
return 0 |
|||
} |
|||
return data[i] |
|||
} |
|||
|
|||
func max(data []int64)int64{ |
|||
return data[len(data)-1] |
|||
} |
|||
|
|||
func statistics(data []int64)(int64, int64, int64){ |
|||
a := avg(data) |
|||
sort.Sort(SortInt64(data)) |
|||
m := percent50(data) |
|||
p9 := percent90(data) |
|||
p95 := percent95(data) |
|||
ma := max(data) |
|||
fmt.Printf("\t avg: %d middle: %dms percent90: %dms percent95: %dms max: %dms\n", a, m, p9, p95, ma) |
|||
return a, p95, ma |
|||
} |
|||
|
|||
func main(){ |
|||
var tcase, count int |
|||
var host, ak, sk, dir string |
|||
flag.IntVar(&tcase, "case", 1, "which test case to use") |
|||
flag.IntVar(&count, "count", 100, "total test count") |
|||
flag.StringVar(&host, "host", "http://testspeed.xk.design", "access host") |
|||
flag.StringVar(&ak, "ak", "", "access key") |
|||
flag.StringVar(&sk, "sk", "", "access secret") |
|||
flag.StringVar(&dir , "dir", "", "img path") |
|||
flag.Parse() |
|||
switch tcase { |
|||
case 1: |
|||
for _, f := range test1 { |
|||
test(f, count, host) |
|||
testResize(f, count, host) |
|||
} |
|||
case 2: |
|||
for _, f := range test2 { |
|||
test(f, count, host) |
|||
testResize(f, count, host) |
|||
} |
|||
case 3: |
|||
case3(dir, ak, sk, count, host) |
|||
} |
|||
} |
@ -0,0 +1,31 @@ |
|||
<!DOCTYPE html> |
|||
<html> |
|||
<head> |
|||
<meta charset="UTF-8"> |
|||
<!-- import CSS --> |
|||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@4.6.2/dist/css/bootstrap.min.css" integrity="sha384-xOolHFLEh07PJGoPkLv1IbcEPTNtaed2xpHsD9ESMhqIYd0nLMwNLD69Npy4HI+N" crossorigin="anonymous"> |
|||
<title>{{.title}}</title> |
|||
</head> |
|||
<body> |
|||
<div class="jumbotron jumbotron-fluid" style="height:100vh" > |
|||
<div class="container"> |
|||
<div class=""> |
|||
<h3 style="text-align: center;">{{.title}}</h3> |
|||
</div> |
|||
<hr class="my-4" > |
|||
{{ range .files }} |
|||
<div class="row" style=""> |
|||
<div style="width:80%;" class="p-3 mb-2 bg-light text-dark"><a href="/ui/{{.}}">{{.}}</a></div> |
|||
</div> |
|||
{{else}} |
|||
<div class="" style=""> |
|||
<p>这家伙很懒,还没有发布任何信息</p> |
|||
</div> |
|||
{{end}} |
|||
</div> |
|||
</div> |
|||
<!-- import JavaScript --> |
|||
<script src="https://cdn.jsdelivr.net/npm/jquery@3.5.1/dist/jquery.slim.min.js" integrity="sha384-DfXdz2htPH0lsSSs5nCTpuj/zy4C+OGpamoFVy38MVBnE+IbbVYUew+OrCXaRkfj" crossorigin="anonymous"></script> |
|||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@4.6.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-Fy6S3B9q64WdZWQUiU+q4/2Lc9npb8tCaSX9FK7E8HnRr0Jz8D6OP9dO5Vg3Q9ct" crossorigin="anonymous"></script> |
|||
</body> |
|||
</html> |
File diff suppressed because one or more lines are too long
Loading…
Reference in new issue