generated from bing/readnotes
538 lines
12 KiB
Go
538 lines
12 KiB
Go
package main
|
||
|
||
import (
|
||
"bytes"
|
||
"flag"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"os"
|
||
"path"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||
)
|
||
|
||
const (
|
||
metaKeyUpdatedAt = "UpdatedAt"
|
||
metaKeyModifiedAt = "Last-Modified"
|
||
metaKeyFileSize = "Content-Length"
|
||
)
|
||
|
||
type AliOSS struct {
|
||
bucket *oss.Bucket
|
||
client *oss.Client
|
||
endpoint string
|
||
accessKey string
|
||
secret string
|
||
bucketName string
|
||
}
|
||
|
||
func NewAliOSS(endpoint, accessKey, secret string, bucket string) *AliOSS {
|
||
return &AliOSS{
|
||
endpoint: endpoint,
|
||
accessKey: accessKey,
|
||
secret: secret,
|
||
bucketName: bucket,
|
||
}
|
||
}
|
||
|
||
func (a *AliOSS) Init(opts ...oss.ClientOption) error {
|
||
client, err := oss.New(a.endpoint, a.accessKey, a.secret, opts...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
a.client = client
|
||
a.bucket, err = client.Bucket(a.bucketName)
|
||
return err
|
||
}
|
||
|
||
// Copy 拷贝文件
|
||
func (a *AliOSS) Copy(src, dst string, opts ...oss.Option) error {
|
||
|
||
_, err := a.bucket.CopyObject(src, dst, opts...)
|
||
return err
|
||
}
|
||
|
||
func (a *AliOSS) Delete(f []string, opts ...oss.Option) ([]string, error) {
|
||
res, err := a.bucket.DeleteObjects(f, opts...)
|
||
return res.DeletedObjects, err
|
||
}
|
||
|
||
// Download 下载文件
|
||
func (a *AliOSS) Download(obj, savePath string, opts ...oss.Option) error {
|
||
err := a.bucket.GetObjectToFile(obj, savePath, opts...)
|
||
return err
|
||
}
|
||
|
||
// IsExist 判断文件是否存在
|
||
func (a *AliOSS) IsExist(obj string) (bool, error) {
|
||
return a.bucket.IsObjectExist(obj)
|
||
}
|
||
|
||
func (a *AliOSS) UploadObject(dest string, src io.Reader, opts ...oss.Option) error {
|
||
return a.bucket.PutObject(dest, src, opts...)
|
||
}
|
||
|
||
func (a *AliOSS) UploadByte(dest string, buf []byte, opts ...oss.Option) error {
|
||
return a.UploadObject(dest, bytes.NewReader(buf), opts...)
|
||
}
|
||
|
||
func (a *AliOSS) UploadStream(dest string, src *os.File, opts ...oss.Option) error {
|
||
// 读取本地文件。
|
||
return a.UploadObject(dest, src, opts...)
|
||
}
|
||
|
||
func (a *AliOSS) UploadFile(dest string, filePath string, opts ...oss.Option) error {
|
||
err := a.bucket.PutObjectFromFile(dest, filePath, opts...)
|
||
return err
|
||
}
|
||
|
||
// set meta
|
||
func (a *AliOSS) SetObjectMeta(file string, key, val string) error {
|
||
meta := oss.Meta(key, val)
|
||
err := a.bucket.SetObjectMeta(file, meta)
|
||
return err
|
||
}
|
||
|
||
// set metas
|
||
func (a *AliOSS) SetObjectMetaMap(file string, metaMap map[string]string) error {
|
||
metas := []oss.Option{}
|
||
for k, v := range metaMap {
|
||
metas = append(metas, oss.Meta(k, v))
|
||
}
|
||
if len(metas) == 0 {
|
||
return nil
|
||
}
|
||
return a.bucket.SetObjectMeta(file, metas...)
|
||
}
|
||
|
||
// get metas
|
||
func (a *AliOSS) GetObjectMeta(file string) (map[string]string, error) {
|
||
res := make(map[string]string)
|
||
metas, err := a.bucket.GetObjectMeta(file)
|
||
for k, v := range metas {
|
||
val := ""
|
||
if len(v) > 0 {
|
||
val = v[0]
|
||
}
|
||
res[k] = val
|
||
}
|
||
return res, err
|
||
}
|
||
|
||
// set updatedAt
|
||
func (a *AliOSS) SetUpdatedAt(file string, updatedAt *time.Time) error {
|
||
if updatedAt == nil {
|
||
now := time.Now()
|
||
updatedAt = &now
|
||
}
|
||
return a.SetObjectMeta(file, metaKeyUpdatedAt, strconv.FormatInt(updatedAt.Unix(), 10))
|
||
}
|
||
|
||
// get updatedAt
|
||
func (a *AliOSS) GetUpdatedAt(file string) (int64, error) {
|
||
metas, err := a.GetObjectMeta(file)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
if u, ok := metas[metaKeyUpdatedAt]; ok {
|
||
i, err := strconv.ParseInt(u, 10, 64)
|
||
return i, err
|
||
}
|
||
return 0, nil
|
||
}
|
||
|
||
// get updatedAt
|
||
func (a *AliOSS) GetModifiedAt(file string) (int64, error) {
|
||
metas, err := a.GetObjectMeta(file)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
if u, ok := metas[metaKeyModifiedAt]; ok {
|
||
//Thu, 07 Apr 2022 03:13:49 GMT
|
||
t, err := time.Parse(time.RFC1123, u)
|
||
return t.Unix(), err
|
||
}
|
||
return 0, nil
|
||
}
|
||
|
||
// upload stream with set updated at
|
||
func (a *AliOSS) UploadObjectWithUpdatedAt(dest string, src io.Reader, opts ...oss.Option) error {
|
||
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
|
||
opts = append(opts, opt)
|
||
return a.UploadObject(dest, src, opts...)
|
||
}
|
||
|
||
// upload file with set updated at
|
||
func (a *AliOSS) UploadFileWithUpdatedAt(dest string, src string, opts ...oss.Option) error {
|
||
err := a.UploadFile(dest, src, opts...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return a.SetUpdatedAt(dest, nil)
|
||
}
|
||
|
||
// upload bytes with set updated at
|
||
func (a *AliOSS) UploadBytesWithUpdatedAt(dest string, src []byte, opts ...oss.Option) error {
|
||
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
|
||
opts = append(opts, opt)
|
||
return a.UploadByte(dest, src, opts...)
|
||
}
|
||
|
||
func (a *AliOSS) Name() string {
|
||
return a.bucketName
|
||
}
|
||
|
||
// check file version, if not need download ,return true
|
||
func (a *AliOSS) CheckFileVersion(obj string, info os.FileInfo) (bool, error) {
|
||
metas, err := a.GetObjectMeta(obj)
|
||
if err != nil {
|
||
return true, err
|
||
}
|
||
// var modifiedAt time.Time
|
||
var modifiedAt int64
|
||
if u, ok := metas[metaKeyModifiedAt]; ok {
|
||
//Thu, 07 Apr 2022 03:13:49 GMT
|
||
t, err := time.Parse(time.RFC1123, u)
|
||
if err == nil {
|
||
modifiedAt = t.Unix()
|
||
}
|
||
// modifiedAt, _ = strconv.ParseInt(u, 10, 64)
|
||
}
|
||
//TODO is need to check file size??
|
||
// var length int64 = 0
|
||
// if u, ok := metas[metaKeyFileSize]; ok {
|
||
// n, err := strconv.ParseInt(u, 10, 64)
|
||
// if err == nil {
|
||
// length = n
|
||
// }
|
||
// }
|
||
//如果本地文件时间更靠后或者本地文件更大，返回true
|
||
// return info.ModTime().Unix() >= modifiedAt || length <= info.Size(), nil
|
||
//如果本地文件时间更靠后，返回true, 防止意外退出，未成功上传
|
||
return info.ModTime().Unix() >= modifiedAt, nil
|
||
}
|
||
|
||
|
||
// const host = "http://testspeed.xk.design"
|
||
|
||
const MAX = 300
|
||
|
||
type file struct{
|
||
Name string
|
||
Width int
|
||
Height int
|
||
Src string
|
||
}
|
||
|
||
var test1 = []file{
|
||
{
|
||
Name: "test4k.jpg",
|
||
Width: 0,
|
||
Height: 0,
|
||
Src: "test4k.jpg",
|
||
},
|
||
{
|
||
Name: "test2k.jpg",
|
||
Width: 2590,
|
||
Height: 1619,
|
||
Src: "test4k.jpg",
|
||
},
|
||
{
|
||
Name: "test1k.jpg",
|
||
Width: 1831,
|
||
Height: 1144,
|
||
Src: "test4k.jpg",
|
||
},
|
||
{
|
||
Name: "test600.jpg",
|
||
Width: 647,
|
||
Height: 404,
|
||
Src: "test4k.jpg",
|
||
},
|
||
{
|
||
Name: "test1of10.jpg",
|
||
Width: 384,
|
||
Height: 240,
|
||
Src: "test4k.jpg",
|
||
},
|
||
}
|
||
|
||
var test2 = []file{
|
||
{
|
||
Name: "8ktestsrc.jpeg",
|
||
Width: 7680,
|
||
Height: 4320,
|
||
Src: "8ktestsrc.jpeg",
|
||
},
|
||
{
|
||
Name: "8ktest3k.jpeg",
|
||
Width: 2730,
|
||
Height: 1536,
|
||
Src: "8ktestsrc.jpeg",
|
||
},
|
||
{
|
||
Name: "8ktest1080.jpeg",
|
||
Width: 1930,
|
||
Height: 1086,
|
||
Src: "8ktestsrc.jpeg",
|
||
},
|
||
{
|
||
Name: "8ktest682.jpeg",
|
||
Width: 682,
|
||
Height: 384,
|
||
Src: "8ktestsrc.jpeg",
|
||
},
|
||
{
|
||
Name: "8ktest400.jpeg",
|
||
Width: 400,
|
||
Height: 225,
|
||
Src: "8ktestsrc.jpeg",
|
||
},
|
||
{
|
||
Name: "8ktest300.jpeg",
|
||
Width: 300,
|
||
Height: 300,
|
||
Src: "8ktestsrc.jpeg",
|
||
},
|
||
}
|
||
|
||
func NewRequest(url string)(*http.Request, error){
|
||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Add("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||
req.Header.Add("Accept-Encoding","gzip, deflate")
|
||
req.Header.Add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56")
|
||
req.Header.Add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
|
||
req.Header.Add("Cache-Control","no-cache")
|
||
req.Header.Add("Connection"," keep-alive")
|
||
req.Header.Add("Cookie"," Hm_lvt_2e2c80c1bd8a51afc2d7de641330a397=1669089775,1669615598,1669704782,1669875251; Hm_lpvt_2e2c80c1bd8a51afc2d7de641330a397=1669964998")
|
||
req.Header.Add("Host"," test-pic.xk.design")
|
||
// req.Header.Add("If-Modified-Since"," Fri, 02 Dec 2022 06:31:28 GMT")
|
||
// req.Header.Add("If-None-Match","\"4B8579BA5E07DD57405973606E777476\"")
|
||
// req.Header.Add("Upgrade-Insecure-Requests","1")
|
||
return req, nil
|
||
}
|
||
|
||
|
||
func test(f file, count int, host string){
|
||
fmt.Printf("begin normal testing for %s\n", f.Name)
|
||
startedAt := time.Now().UnixMilli()
|
||
errCount := 0
|
||
costMS := []int64{}
|
||
for i := 0; i < count; i ++ {
|
||
req, err := NewRequest(host + "/" + f.Name)
|
||
if err != nil {
|
||
errCount ++
|
||
continue
|
||
}
|
||
client := &http.Client{}
|
||
begin := time.Now().UnixMilli()
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
errCount++
|
||
continue
|
||
}
|
||
defer resp.Body.Close()
|
||
data, err := io.ReadAll(resp.Body)
|
||
os.WriteFile("/dev/null", data, os.ModePerm)
|
||
if err != nil {
|
||
errCount++
|
||
continue
|
||
}
|
||
end := time.Now().UnixMilli()
|
||
costMS = append(costMS, end-begin)
|
||
}
|
||
finishedAt := time.Now().UnixMilli()
|
||
totalCost := finishedAt - startedAt
|
||
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
|
||
statistics(costMS)
|
||
}
|
||
|
||
func testResize(f file, count int, host string)(int64, int64, int64){
|
||
fmt.Printf("begin resize testing for %s\n", f.Name)
|
||
startedAt := time.Now().UnixMilli()
|
||
errCount := 0
|
||
costMS := []int64{}
|
||
for i := 0; i < count; i ++ {
|
||
query := "?x-oss-process=image/resize,m_lfit,w_"+strconv.Itoa(f.Width) + ",h_" + strconv.Itoa(f.Height)
|
||
if f.Height == 0 || f.Width == 0 {
|
||
query = ""
|
||
}
|
||
req, err := NewRequest(host + "/" + f.Src + query)
|
||
if err != nil {
|
||
errCount++
|
||
continue
|
||
}
|
||
client := http.Client{}
|
||
begin := time.Now().UnixMilli()
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
errCount++
|
||
continue
|
||
}
|
||
defer resp.Body.Close()
|
||
data, err := io.ReadAll(resp.Body)
|
||
os.WriteFile("/dev/null", data, os.ModePerm)
|
||
if err != nil {
|
||
errCount++
|
||
continue
|
||
}
|
||
end := time.Now().UnixMilli()
|
||
costMS = append(costMS, end-begin)
|
||
// if resp != nil {
|
||
// fmt.Println(resp , resp.StatusCode, resp.Status)
|
||
// }
|
||
}
|
||
finishedAt := time.Now().UnixMilli()
|
||
totalCost := finishedAt - startedAt
|
||
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
|
||
return statistics(costMS)
|
||
}
|
||
|
||
func filelist(dir string)([]string, error){
|
||
fis, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
fs := []string{}
|
||
for _, f := range fis {
|
||
if !f.IsDir() {
|
||
fs = append(fs, f.Name())
|
||
}
|
||
}
|
||
return fs, nil
|
||
}
|
||
|
||
|
||
func case3(dir string, ak string, sk string, count int, host string)error{
|
||
files, err := filelist(dir)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
alioss := NewAliOSS("oss-cn-hangzhou.aliyuncs.com", ak,sk, "oxslmimg")
|
||
alioss.Init()
|
||
avgmap := make(map[string][]string)
|
||
p95map := make(map[string][]string)
|
||
for _, f := range files {
|
||
err = alioss.UploadFile(f, path.Join(dir,f))
|
||
if err != nil {
|
||
fmt.Println("upload failed:", err, f, path.Join(dir,f))
|
||
continue
|
||
}
|
||
avgmap[f] = []string{}
|
||
p95map[f] = []string{}
|
||
for _, t := range test1 {
|
||
t.Src = f
|
||
t.Name = f
|
||
avg, p95, _ := testResize(t, count, host)
|
||
avgmap[f] = append(avgmap[f], strconv.FormatInt(avg, 10))
|
||
p95map[f] = append(p95map[f], strconv.FormatInt(p95, 10))
|
||
}
|
||
}
|
||
fmt.Println("")
|
||
for f, v := range avgmap {
|
||
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
|
||
}
|
||
for f, v := range p95map {
|
||
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func avg(data []int64)int64{
|
||
if len(data) == 0 {
|
||
return 0
|
||
}
|
||
total := int64(0)
|
||
for _, i := range data {
|
||
total = total + i
|
||
}
|
||
return total/int64(len(data))
|
||
}
|
||
|
||
type SortInt64 []int64
|
||
|
||
func (a SortInt64) Len() int { return len(a) }
|
||
func (a SortInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||
func (a SortInt64) Less(i, j int) bool { return a[i] < a[j] }
|
||
|
||
func percent50(data []int64)int64{
|
||
if len(data) == 0 {
|
||
return 0
|
||
}
|
||
i := int(float64(len(data)) * 0.5 )
|
||
if i == len(data) {
|
||
return 0
|
||
}
|
||
return data[i]
|
||
}
|
||
|
||
func percent90(data []int64)int64{
|
||
if len(data) == 0 {
|
||
return 0
|
||
}
|
||
i := int(float64(len(data)) * 0.9 )
|
||
if i == len(data) {
|
||
return 0
|
||
}
|
||
return data[i]
|
||
}
|
||
|
||
func percent95(data []int64)int64{
|
||
if len(data) == 0 {
|
||
return 0
|
||
}
|
||
i := int(float64(len(data)) * 0.95 )
|
||
if i == len(data) {
|
||
return 0
|
||
}
|
||
return data[i]
|
||
}
|
||
|
||
func max(data []int64)int64{
|
||
return data[len(data)-1]
|
||
}
|
||
|
||
func statistics(data []int64)(int64, int64, int64){
|
||
a := avg(data)
|
||
sort.Sort(SortInt64(data))
|
||
m := percent50(data)
|
||
p9 := percent90(data)
|
||
p95 := percent95(data)
|
||
ma := max(data)
|
||
fmt.Printf("\t avg: %d middle: %dms percent90: %dms percent95: %dms max: %dms\n", a, m, p9, p95, ma)
|
||
return a, p95, ma
|
||
}
|
||
|
||
func main(){
|
||
var tcase, count int
|
||
var host, ak, sk, dir string
|
||
flag.IntVar(&tcase, "case", 1, "which test case to use")
|
||
flag.IntVar(&count, "count", 100, "total test count")
|
||
flag.StringVar(&host, "host", "http://testspeed.xk.design", "access host")
|
||
flag.StringVar(&ak, "ak", "", "access key")
|
||
flag.StringVar(&sk, "sk", "", "access secret")
|
||
flag.StringVar(&dir , "dir", "", "img path")
|
||
flag.Parse()
|
||
switch tcase {
|
||
case 1:
|
||
for _, f := range test1 {
|
||
test(f, count, host)
|
||
testResize(f, count, host)
|
||
}
|
||
case 2:
|
||
for _, f := range test2 {
|
||
test(f, count, host)
|
||
testResize(f, count, host)
|
||
}
|
||
case 3:
|
||
case3(dir, ak, sk, count, host)
|
||
}
|
||
} |