You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
538 lines
12 KiB
538 lines
12 KiB
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
|
)
|
|
|
|
const (
|
|
metaKeyUpdatedAt = "UpdatedAt"
|
|
metaKeyModifiedAt = "Last-Modified"
|
|
metaKeyFileSize = "Content-Length"
|
|
)
|
|
|
|
type AliOSS struct {
|
|
bucket *oss.Bucket
|
|
client *oss.Client
|
|
endpoint string
|
|
accessKey string
|
|
secret string
|
|
bucketName string
|
|
}
|
|
|
|
func NewAliOSS(endpoint, accessKey, secret string, bucket string) *AliOSS {
|
|
return &AliOSS{
|
|
endpoint: endpoint,
|
|
accessKey: accessKey,
|
|
secret: secret,
|
|
bucketName: bucket,
|
|
}
|
|
}
|
|
|
|
func (a *AliOSS) Init(opts ...oss.ClientOption) error {
|
|
client, err := oss.New(a.endpoint, a.accessKey, a.secret, opts...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
a.client = client
|
|
a.bucket, err = client.Bucket(a.bucketName)
|
|
return err
|
|
}
|
|
|
|
// Copy 拷贝文件
|
|
func (a *AliOSS) Copy(src, dst string, opts ...oss.Option) error {
|
|
|
|
_, err := a.bucket.CopyObject(src, dst, opts...)
|
|
return err
|
|
}
|
|
|
|
func (a *AliOSS) Delete(f []string, opts ...oss.Option) ([]string, error) {
|
|
res, err := a.bucket.DeleteObjects(f, opts...)
|
|
return res.DeletedObjects, err
|
|
}
|
|
|
|
// Download 下载文件
|
|
func (a *AliOSS) Download(obj, savePath string, opts ...oss.Option) error {
|
|
err := a.bucket.GetObjectToFile(obj, savePath, opts...)
|
|
return err
|
|
}
|
|
|
|
// IsExist 判断文件是否存在
|
|
func (a *AliOSS) IsExist(obj string) (bool, error) {
|
|
return a.bucket.IsObjectExist(obj)
|
|
}
|
|
|
|
func (a *AliOSS) UploadObject(dest string, src io.Reader, opts ...oss.Option) error {
|
|
return a.bucket.PutObject(dest, src, opts...)
|
|
}
|
|
|
|
func (a *AliOSS) UploadByte(dest string, buf []byte, opts ...oss.Option) error {
|
|
return a.UploadObject(dest, bytes.NewReader(buf), opts...)
|
|
}
|
|
|
|
func (a *AliOSS) UploadStream(dest string, src *os.File, opts ...oss.Option) error {
|
|
// 读取本地文件。
|
|
return a.UploadObject(dest, src, opts...)
|
|
}
|
|
|
|
func (a *AliOSS) UploadFile(dest string, filePath string, opts ...oss.Option) error {
|
|
err := a.bucket.PutObjectFromFile(dest, filePath, opts...)
|
|
return err
|
|
}
|
|
|
|
// set meta
|
|
func (a *AliOSS) SetObjectMeta(file string, key, val string) error {
|
|
meta := oss.Meta(key, val)
|
|
err := a.bucket.SetObjectMeta(file, meta)
|
|
return err
|
|
}
|
|
|
|
// set metas
|
|
func (a *AliOSS) SetObjectMetaMap(file string, metaMap map[string]string) error {
|
|
metas := []oss.Option{}
|
|
for k, v := range metaMap {
|
|
metas = append(metas, oss.Meta(k, v))
|
|
}
|
|
if len(metas) == 0 {
|
|
return nil
|
|
}
|
|
return a.bucket.SetObjectMeta(file, metas...)
|
|
}
|
|
|
|
// get metas
|
|
func (a *AliOSS) GetObjectMeta(file string) (map[string]string, error) {
|
|
res := make(map[string]string)
|
|
metas, err := a.bucket.GetObjectMeta(file)
|
|
for k, v := range metas {
|
|
val := ""
|
|
if len(v) > 0 {
|
|
val = v[0]
|
|
}
|
|
res[k] = val
|
|
}
|
|
return res, err
|
|
}
|
|
|
|
// set updatedAt
|
|
func (a *AliOSS) SetUpdatedAt(file string, updatedAt *time.Time) error {
|
|
if updatedAt == nil {
|
|
now := time.Now()
|
|
updatedAt = &now
|
|
}
|
|
return a.SetObjectMeta(file, metaKeyUpdatedAt, strconv.FormatInt(updatedAt.Unix(), 10))
|
|
}
|
|
|
|
// get updatedAt
|
|
func (a *AliOSS) GetUpdatedAt(file string) (int64, error) {
|
|
metas, err := a.GetObjectMeta(file)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if u, ok := metas[metaKeyUpdatedAt]; ok {
|
|
i, err := strconv.ParseInt(u, 10, 64)
|
|
return i, err
|
|
}
|
|
return 0, nil
|
|
}
|
|
|
|
// get updatedAt
|
|
func (a *AliOSS) GetModifiedAt(file string) (int64, error) {
|
|
metas, err := a.GetObjectMeta(file)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if u, ok := metas[metaKeyModifiedAt]; ok {
|
|
//Thu, 07 Apr 2022 03:13:49 GMT
|
|
t, err := time.Parse(time.RFC1123, u)
|
|
return t.Unix(), err
|
|
}
|
|
return 0, nil
|
|
}
|
|
|
|
// upload stream with set updated at
|
|
func (a *AliOSS) UploadObjectWithUpdatedAt(dest string, src io.Reader, opts ...oss.Option) error {
|
|
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
|
|
opts = append(opts, opt)
|
|
return a.UploadObject(dest, src, opts...)
|
|
}
|
|
|
|
// upload file with set updated at
|
|
func (a *AliOSS) UploadFileWithUpdatedAt(dest string, src string, opts ...oss.Option) error {
|
|
err := a.UploadFile(dest, src, opts...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return a.SetUpdatedAt(dest, nil)
|
|
}
|
|
|
|
// upload bytes with set updated at
|
|
func (a *AliOSS) UploadBytesWithUpdatedAt(dest string, src []byte, opts ...oss.Option) error {
|
|
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
|
|
opts = append(opts, opt)
|
|
return a.UploadByte(dest, src, opts...)
|
|
}
|
|
|
|
func (a *AliOSS) Name() string {
|
|
return a.bucketName
|
|
}
|
|
|
|
// check file version, if not need download ,return true
|
|
func (a *AliOSS) CheckFileVersion(obj string, info os.FileInfo) (bool, error) {
|
|
metas, err := a.GetObjectMeta(obj)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
// var modifiedAt time.Time
|
|
var modifiedAt int64
|
|
if u, ok := metas[metaKeyModifiedAt]; ok {
|
|
//Thu, 07 Apr 2022 03:13:49 GMT
|
|
t, err := time.Parse(time.RFC1123, u)
|
|
if err == nil {
|
|
modifiedAt = t.Unix()
|
|
}
|
|
// modifiedAt, _ = strconv.ParseInt(u, 10, 64)
|
|
}
|
|
//TODO is need to check file size??
|
|
// var length int64 = 0
|
|
// if u, ok := metas[metaKeyFileSize]; ok {
|
|
// n, err := strconv.ParseInt(u, 10, 64)
|
|
// if err == nil {
|
|
// length = n
|
|
// }
|
|
// }
|
|
//如果本地文件时间更靠后或者本地文件更大，返回true
|
|
// return info.ModTime().Unix() >= modifiedAt || length <= info.Size(), nil
|
|
//如果本地文件时间更靠后，返回true, 防止意外退出，未成功上传
|
|
return info.ModTime().Unix() >= modifiedAt, nil
|
|
}
|
|
|
|
|
|
// const host = "http://testspeed.xk.design"
|
|
|
|
const MAX = 300
|
|
|
|
type file struct{
|
|
Name string
|
|
Width int
|
|
Height int
|
|
Src string
|
|
}
|
|
|
|
var test1 = []file{
|
|
{
|
|
Name: "test4k.jpg",
|
|
Width: 0,
|
|
Height: 0,
|
|
Src: "test4k.jpg",
|
|
},
|
|
{
|
|
Name: "test2k.jpg",
|
|
Width: 2590,
|
|
Height: 1619,
|
|
Src: "test4k.jpg",
|
|
},
|
|
{
|
|
Name: "test1k.jpg",
|
|
Width: 1831,
|
|
Height: 1144,
|
|
Src: "test4k.jpg",
|
|
},
|
|
{
|
|
Name: "test600.jpg",
|
|
Width: 647,
|
|
Height: 404,
|
|
Src: "test4k.jpg",
|
|
},
|
|
{
|
|
Name: "test1of10.jpg",
|
|
Width: 384,
|
|
Height: 240,
|
|
Src: "test4k.jpg",
|
|
},
|
|
}
|
|
|
|
var test2 = []file{
|
|
{
|
|
Name: "8ktestsrc.jpeg",
|
|
Width: 7680,
|
|
Height: 4320,
|
|
Src: "8ktestsrc.jpeg",
|
|
},
|
|
{
|
|
Name: "8ktest3k.jpeg",
|
|
Width: 2730,
|
|
Height: 1536,
|
|
Src: "8ktestsrc.jpeg",
|
|
},
|
|
{
|
|
Name: "8ktest1080.jpeg",
|
|
Width: 1930,
|
|
Height: 1086,
|
|
Src: "8ktestsrc.jpeg",
|
|
},
|
|
{
|
|
Name: "8ktest682.jpeg",
|
|
Width: 682,
|
|
Height: 384,
|
|
Src: "8ktestsrc.jpeg",
|
|
},
|
|
{
|
|
Name: "8ktest400.jpeg",
|
|
Width: 400,
|
|
Height: 225,
|
|
Src: "8ktestsrc.jpeg",
|
|
},
|
|
{
|
|
Name: "8ktest300.jpeg",
|
|
Width: 300,
|
|
Height: 300,
|
|
Src: "8ktestsrc.jpeg",
|
|
},
|
|
}
|
|
|
|
func NewRequest(url string)(*http.Request, error){
|
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Add("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
|
req.Header.Add("Accept-Encoding","gzip, deflate")
|
|
req.Header.Add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56")
|
|
req.Header.Add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
|
|
req.Header.Add("Cache-Control","no-cache")
|
|
req.Header.Add("Connection"," keep-alive")
|
|
req.Header.Add("Cookie"," Hm_lvt_2e2c80c1bd8a51afc2d7de641330a397=1669089775,1669615598,1669704782,1669875251; Hm_lpvt_2e2c80c1bd8a51afc2d7de641330a397=1669964998")
|
|
req.Header.Add("Host"," test-pic.xk.design")
|
|
// req.Header.Add("If-Modified-Since"," Fri, 02 Dec 2022 06:31:28 GMT")
|
|
// req.Header.Add("If-None-Match","\"4B8579BA5E07DD57405973606E777476\"")
|
|
// req.Header.Add("Upgrade-Insecure-Requests","1")
|
|
return req, nil
|
|
}
|
|
|
|
|
|
func test(f file, count int, host string){
|
|
fmt.Printf("begin normal testing for %s\n", f.Name)
|
|
startedAt := time.Now().UnixMilli()
|
|
errCount := 0
|
|
costMS := []int64{}
|
|
for i := 0; i < count; i ++ {
|
|
req, err := NewRequest(host + "/" + f.Name)
|
|
if err != nil {
|
|
errCount ++
|
|
continue
|
|
}
|
|
client := &http.Client{}
|
|
begin := time.Now().UnixMilli()
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
errCount++
|
|
continue
|
|
}
|
|
defer resp.Body.Close()
|
|
data, err := io.ReadAll(resp.Body)
|
|
os.WriteFile("/dev/null", data, os.ModePerm)
|
|
if err != nil {
|
|
errCount++
|
|
continue
|
|
}
|
|
end := time.Now().UnixMilli()
|
|
costMS = append(costMS, end-begin)
|
|
}
|
|
finishedAt := time.Now().UnixMilli()
|
|
totalCost := finishedAt - startedAt
|
|
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
|
|
statistics(costMS)
|
|
}
|
|
|
|
func testResize(f file, count int, host string)(int64, int64, int64){
|
|
fmt.Printf("begin resize testing for %s\n", f.Name)
|
|
startedAt := time.Now().UnixMilli()
|
|
errCount := 0
|
|
costMS := []int64{}
|
|
for i := 0; i < count; i ++ {
|
|
query := "?x-oss-process=image/resize,m_lfit,w_"+strconv.Itoa(f.Width) + ",h_" + strconv.Itoa(f.Height)
|
|
if f.Height == 0 || f.Width == 0 {
|
|
query = ""
|
|
}
|
|
req, err := NewRequest(host + "/" + f.Src + query)
|
|
if err != nil {
|
|
errCount++
|
|
continue
|
|
}
|
|
client := http.Client{}
|
|
begin := time.Now().UnixMilli()
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
errCount++
|
|
continue
|
|
}
|
|
defer resp.Body.Close()
|
|
data, err := io.ReadAll(resp.Body)
|
|
os.WriteFile("/dev/null", data, os.ModePerm)
|
|
if err != nil {
|
|
errCount++
|
|
continue
|
|
}
|
|
end := time.Now().UnixMilli()
|
|
costMS = append(costMS, end-begin)
|
|
// if resp != nil {
|
|
// fmt.Println(resp , resp.StatusCode, resp.Status)
|
|
// }
|
|
}
|
|
finishedAt := time.Now().UnixMilli()
|
|
totalCost := finishedAt - startedAt
|
|
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
|
|
return statistics(costMS)
|
|
}
|
|
|
|
func filelist(dir string)([]string, error){
|
|
fis, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
fs := []string{}
|
|
for _, f := range fis {
|
|
if !f.IsDir() {
|
|
fs = append(fs, f.Name())
|
|
}
|
|
}
|
|
return fs, nil
|
|
}
|
|
|
|
|
|
func case3(dir string, ak string, sk string, count int, host string)error{
|
|
files, err := filelist(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
alioss := NewAliOSS("oss-cn-hangzhou.aliyuncs.com", ak,sk, "oxslmimg")
|
|
alioss.Init()
|
|
avgmap := make(map[string][]string)
|
|
p95map := make(map[string][]string)
|
|
for _, f := range files {
|
|
err = alioss.UploadFile(f, path.Join(dir,f))
|
|
if err != nil {
|
|
fmt.Println("upload failed:", err, f, path.Join(dir,f))
|
|
continue
|
|
}
|
|
avgmap[f] = []string{}
|
|
p95map[f] = []string{}
|
|
for _, t := range test1 {
|
|
t.Src = f
|
|
t.Name = f
|
|
avg, p95, _ := testResize(t, count, host)
|
|
avgmap[f] = append(avgmap[f], strconv.FormatInt(avg, 10))
|
|
p95map[f] = append(p95map[f], strconv.FormatInt(p95, 10))
|
|
}
|
|
}
|
|
fmt.Println("")
|
|
for f, v := range avgmap {
|
|
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
|
|
}
|
|
for f, v := range p95map {
|
|
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func avg(data []int64)int64{
|
|
if len(data) == 0 {
|
|
return 0
|
|
}
|
|
total := int64(0)
|
|
for _, i := range data {
|
|
total = total + i
|
|
}
|
|
return total/int64(len(data))
|
|
}
|
|
|
|
type SortInt64 []int64
|
|
|
|
func (a SortInt64) Len() int { return len(a) }
|
|
func (a SortInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
func (a SortInt64) Less(i, j int) bool { return a[i] < a[j] }
|
|
|
|
func percent50(data []int64)int64{
|
|
if len(data) == 0 {
|
|
return 0
|
|
}
|
|
i := int(float64(len(data)) * 0.5 )
|
|
if i == len(data) {
|
|
return 0
|
|
}
|
|
return data[i]
|
|
}
|
|
|
|
func percent90(data []int64)int64{
|
|
if len(data) == 0 {
|
|
return 0
|
|
}
|
|
i := int(float64(len(data)) * 0.9 )
|
|
if i == len(data) {
|
|
return 0
|
|
}
|
|
return data[i]
|
|
}
|
|
|
|
func percent95(data []int64)int64{
|
|
if len(data) == 0 {
|
|
return 0
|
|
}
|
|
i := int(float64(len(data)) * 0.95 )
|
|
if i == len(data) {
|
|
return 0
|
|
}
|
|
return data[i]
|
|
}
|
|
|
|
func max(data []int64)int64{
|
|
return data[len(data)-1]
|
|
}
|
|
|
|
func statistics(data []int64)(int64, int64, int64){
|
|
a := avg(data)
|
|
sort.Sort(SortInt64(data))
|
|
m := percent50(data)
|
|
p9 := percent90(data)
|
|
p95 := percent95(data)
|
|
ma := max(data)
|
|
fmt.Printf("\t avg: %d middle: %dms percent90: %dms percent95: %dms max: %dms\n", a, m, p9, p95, ma)
|
|
return a, p95, ma
|
|
}
|
|
|
|
func main(){
|
|
var tcase, count int
|
|
var host, ak, sk, dir string
|
|
flag.IntVar(&tcase, "case", 1, "which test case to use")
|
|
flag.IntVar(&count, "count", 100, "total test count")
|
|
flag.StringVar(&host, "host", "http://testspeed.xk.design", "access host")
|
|
flag.StringVar(&ak, "ak", "", "access key")
|
|
flag.StringVar(&sk, "sk", "", "access secret")
|
|
flag.StringVar(&dir , "dir", "", "img path")
|
|
flag.Parse()
|
|
switch tcase {
|
|
case 1:
|
|
for _, f := range test1 {
|
|
test(f, count, host)
|
|
testResize(f, count, host)
|
|
}
|
|
case 2:
|
|
for _, f := range test2 {
|
|
test(f, count, host)
|
|
testResize(f, count, host)
|
|
}
|
|
case 3:
|
|
case3(dir, ak, sk, count, host)
|
|
}
|
|
}
|