Browse Source

feat 添加一些小工具

master
bing 2 years ago
parent
commit
c740258b24
  1. 1
      go.mod
  2. 2
      go.sum
  3. 538
      go/oss_image_resize/main.go
  4. 31
      go/uiweb/index.tmpl
  5. 48
      go/uiweb/main.go

1
go.mod

@ -3,6 +3,7 @@ module blog.bing89.com
go 1.17 go 1.17
require ( require (
github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible
github.com/coreos/go-oidc/v3 v3.1.0 github.com/coreos/go-oidc/v3 v3.1.0
github.com/gin-gonic/gin v1.8.1 github.com/gin-gonic/gin v1.8.1
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e

2
go.sum

@ -64,6 +64,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible h1:KXeJoM1wo9I/6xPTyt6qCxoSZnmASiAjlrr0dyTUKt8=
github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=

538
go/oss_image_resize/main.go

@ -0,0 +1,538 @@
package main
import (
"bytes"
"flag"
"fmt"
"io"
"net/http"
"os"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
)
const (
metaKeyUpdatedAt = "UpdatedAt"
metaKeyModifiedAt = "Last-Modified"
metaKeyFileSize = "Content-Length"
)
type AliOSS struct {
bucket *oss.Bucket
client *oss.Client
endpoint string
accessKey string
secret string
bucketName string
}
func NewAliOSS(endpoint, accessKey, secret string, bucket string) *AliOSS {
return &AliOSS{
endpoint: endpoint,
accessKey: accessKey,
secret: secret,
bucketName: bucket,
}
}
func (a *AliOSS) Init(opts ...oss.ClientOption) error {
client, err := oss.New(a.endpoint, a.accessKey, a.secret, opts...)
if err != nil {
return err
}
a.client = client
a.bucket, err = client.Bucket(a.bucketName)
return err
}
// Copy 拷贝文件
func (a *AliOSS) Copy(src, dst string, opts ...oss.Option) error {
_, err := a.bucket.CopyObject(src, dst, opts...)
return err
}
func (a *AliOSS) Delete(f []string, opts ...oss.Option) ([]string, error) {
res, err := a.bucket.DeleteObjects(f, opts...)
return res.DeletedObjects, err
}
// Download 下载文件
func (a *AliOSS) Download(obj, savePath string, opts ...oss.Option) error {
err := a.bucket.GetObjectToFile(obj, savePath, opts...)
return err
}
// IsExist 判断文件是否存在
func (a *AliOSS) IsExist(obj string) (bool, error) {
return a.bucket.IsObjectExist(obj)
}
func (a *AliOSS) UploadObject(dest string, src io.Reader, opts ...oss.Option) error {
return a.bucket.PutObject(dest, src, opts...)
}
func (a *AliOSS) UploadByte(dest string, buf []byte, opts ...oss.Option) error {
return a.UploadObject(dest, bytes.NewReader(buf), opts...)
}
func (a *AliOSS) UploadStream(dest string, src *os.File, opts ...oss.Option) error {
// 读取本地文件。
return a.UploadObject(dest, src, opts...)
}
func (a *AliOSS) UploadFile(dest string, filePath string, opts ...oss.Option) error {
err := a.bucket.PutObjectFromFile(dest, filePath, opts...)
return err
}
// set meta
func (a *AliOSS) SetObjectMeta(file string, key, val string) error {
meta := oss.Meta(key, val)
err := a.bucket.SetObjectMeta(file, meta)
return err
}
// set metas
func (a *AliOSS) SetObjectMetaMap(file string, metaMap map[string]string) error {
metas := []oss.Option{}
for k, v := range metaMap {
metas = append(metas, oss.Meta(k, v))
}
if len(metas) == 0 {
return nil
}
return a.bucket.SetObjectMeta(file, metas...)
}
// get metas
func (a *AliOSS) GetObjectMeta(file string) (map[string]string, error) {
res := make(map[string]string)
metas, err := a.bucket.GetObjectMeta(file)
for k, v := range metas {
val := ""
if len(v) > 0 {
val = v[0]
}
res[k] = val
}
return res, err
}
// set updatedAt
func (a *AliOSS) SetUpdatedAt(file string, updatedAt *time.Time) error {
if updatedAt == nil {
now := time.Now()
updatedAt = &now
}
return a.SetObjectMeta(file, metaKeyUpdatedAt, strconv.FormatInt(updatedAt.Unix(), 10))
}
// get updatedAt
func (a *AliOSS) GetUpdatedAt(file string) (int64, error) {
metas, err := a.GetObjectMeta(file)
if err != nil {
return 0, err
}
if u, ok := metas[metaKeyUpdatedAt]; ok {
i, err := strconv.ParseInt(u, 10, 64)
return i, err
}
return 0, nil
}
// get updatedAt
func (a *AliOSS) GetModifiedAt(file string) (int64, error) {
metas, err := a.GetObjectMeta(file)
if err != nil {
return 0, err
}
if u, ok := metas[metaKeyModifiedAt]; ok {
//Thu, 07 Apr 2022 03:13:49 GMT
t, err := time.Parse(time.RFC1123, u)
return t.Unix(), err
}
return 0, nil
}
// upload stream with set updated at
func (a *AliOSS) UploadObjectWithUpdatedAt(dest string, src io.Reader, opts ...oss.Option) error {
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
opts = append(opts, opt)
return a.UploadObject(dest, src, opts...)
}
// upload file with set updated at
func (a *AliOSS) UploadFileWithUpdatedAt(dest string, src string, opts ...oss.Option) error {
err := a.UploadFile(dest, src, opts...)
if err != nil {
return err
}
return a.SetUpdatedAt(dest, nil)
}
// upload bytes with set updated at
func (a *AliOSS) UploadBytesWithUpdatedAt(dest string, src []byte, opts ...oss.Option) error {
opt := oss.Meta(metaKeyUpdatedAt, strconv.FormatInt(time.Now().Unix(), 10))
opts = append(opts, opt)
return a.UploadByte(dest, src, opts...)
}
func (a *AliOSS) Name() string {
return a.bucketName
}
// check file version, if not need download ,return true
func (a *AliOSS) CheckFileVersion(obj string, info os.FileInfo) (bool, error) {
metas, err := a.GetObjectMeta(obj)
if err != nil {
return true, err
}
// var modifiedAt time.Time
var modifiedAt int64
if u, ok := metas[metaKeyModifiedAt]; ok {
//Thu, 07 Apr 2022 03:13:49 GMT
t, err := time.Parse(time.RFC1123, u)
if err == nil {
modifiedAt = t.Unix()
}
// modifiedAt, _ = strconv.ParseInt(u, 10, 64)
}
//TODO is need to check file size??
// var length int64 = 0
// if u, ok := metas[metaKeyFileSize]; ok {
// n, err := strconv.ParseInt(u, 10, 64)
// if err == nil {
// length = n
// }
// }
//如果本地文件时间更靠后或者本地文件更大,返回true
// return info.ModTime().Unix() >= modifiedAt || length <= info.Size(), nil
//如果本地文件时间更靠后,返回true, 防止意外退出,未成功上传
return info.ModTime().Unix() >= modifiedAt, nil
}
// const host = "http://testspeed.xk.design"
const MAX = 300
type file struct{
Name string
Width int
Height int
Src string
}
var test1 = []file{
{
Name: "test4k.jpg",
Width: 0,
Height: 0,
Src: "test4k.jpg",
},
{
Name: "test2k.jpg",
Width: 2590,
Height: 1619,
Src: "test4k.jpg",
},
{
Name: "test1k.jpg",
Width: 1831,
Height: 1144,
Src: "test4k.jpg",
},
{
Name: "test600.jpg",
Width: 647,
Height: 404,
Src: "test4k.jpg",
},
{
Name: "test1of10.jpg",
Width: 384,
Height: 240,
Src: "test4k.jpg",
},
}
var test2 = []file{
{
Name: "8ktestsrc.jpeg",
Width: 7680,
Height: 4320,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest3k.jpeg",
Width: 2730,
Height: 1536,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest1080.jpeg",
Width: 1930,
Height: 1086,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest682.jpeg",
Width: 682,
Height: 384,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest400.jpeg",
Width: 400,
Height: 225,
Src: "8ktestsrc.jpeg",
},
{
Name: "8ktest300.jpeg",
Width: 300,
Height: 300,
Src: "8ktestsrc.jpeg",
},
}
func NewRequest(url string)(*http.Request, error){
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Add("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
req.Header.Add("Accept-Encoding","gzip, deflate")
req.Header.Add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56")
req.Header.Add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
req.Header.Add("Cache-Control","no-cache")
req.Header.Add("Connection"," keep-alive")
req.Header.Add("Cookie"," Hm_lvt_2e2c80c1bd8a51afc2d7de641330a397=1669089775,1669615598,1669704782,1669875251; Hm_lpvt_2e2c80c1bd8a51afc2d7de641330a397=1669964998")
req.Header.Add("Host"," test-pic.xk.design")
// req.Header.Add("If-Modified-Since"," Fri, 02 Dec 2022 06:31:28 GMT")
// req.Header.Add("If-None-Match","\"4B8579BA5E07DD57405973606E777476\"")
// req.Header.Add("Upgrade-Insecure-Requests","1")
return req, nil
}
func test(f file, count int, host string){
fmt.Printf("begin normal testing for %s\n", f.Name)
startedAt := time.Now().UnixMilli()
errCount := 0
costMS := []int64{}
for i := 0; i < count; i ++ {
req, err := NewRequest(host + "/" + f.Name)
if err != nil {
errCount ++
continue
}
client := &http.Client{}
begin := time.Now().UnixMilli()
resp, err := client.Do(req)
if err != nil {
errCount++
continue
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
os.WriteFile("/dev/null", data, os.ModePerm)
if err != nil {
errCount++
continue
}
end := time.Now().UnixMilli()
costMS = append(costMS, end-begin)
}
finishedAt := time.Now().UnixMilli()
totalCost := finishedAt - startedAt
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
statistics(costMS)
}
func testResize(f file, count int, host string)(int64, int64, int64){
fmt.Printf("begin resize testing for %s\n", f.Name)
startedAt := time.Now().UnixMilli()
errCount := 0
costMS := []int64{}
for i := 0; i < count; i ++ {
query := "?x-oss-process=image/resize,m_lfit,w_"+strconv.Itoa(f.Width) + ",h_" + strconv.Itoa(f.Height)
if f.Height == 0 || f.Width == 0 {
query = ""
}
req, err := NewRequest(host + "/" + f.Src + query)
if err != nil {
errCount++
continue
}
client := http.Client{}
begin := time.Now().UnixMilli()
resp, err := client.Do(req)
if err != nil {
errCount++
continue
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
os.WriteFile("/dev/null", data, os.ModePerm)
if err != nil {
errCount++
continue
}
end := time.Now().UnixMilli()
costMS = append(costMS, end-begin)
// if resp != nil {
// fmt.Println(resp , resp.StatusCode, resp.Status)
// }
}
finishedAt := time.Now().UnixMilli()
totalCost := finishedAt - startedAt
fmt.Printf("in %d tests for file %s@%dx%d, cost %d ms, errors: %d", MAX, f.Src, f.Width, f.Height, totalCost, errCount)
return statistics(costMS)
}
func filelist(dir string)([]string, error){
fis, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
fs := []string{}
for _, f := range fis {
if !f.IsDir() {
fs = append(fs, f.Name())
}
}
return fs, nil
}
func case3(dir string, ak string, sk string, count int, host string)error{
files, err := filelist(dir)
if err != nil {
return err
}
alioss := NewAliOSS("oss-cn-hangzhou.aliyuncs.com", ak,sk, "oxslmimg")
alioss.Init()
avgmap := make(map[string][]string)
p95map := make(map[string][]string)
for _, f := range files {
err = alioss.UploadFile(f, path.Join(dir,f))
if err != nil {
fmt.Println("upload failed:", err, f, path.Join(dir,f))
continue
}
avgmap[f] = []string{}
p95map[f] = []string{}
for _, t := range test1 {
t.Src = f
t.Name = f
avg, p95, _ := testResize(t, count, host)
avgmap[f] = append(avgmap[f], strconv.FormatInt(avg, 10))
p95map[f] = append(p95map[f], strconv.FormatInt(p95, 10))
}
}
fmt.Println("")
for f, v := range avgmap {
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
}
for f, v := range p95map {
fmt.Printf("%s\t%s\n", f, strings.Join(v, "\t"))
}
return nil
}
func avg(data []int64)int64{
if len(data) == 0 {
return 0
}
total := int64(0)
for _, i := range data {
total = total + i
}
return total/int64(len(data))
}
type SortInt64 []int64
func (a SortInt64) Len() int { return len(a) }
func (a SortInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a SortInt64) Less(i, j int) bool { return a[i] < a[j] }
func percent50(data []int64)int64{
if len(data) == 0 {
return 0
}
i := int(float64(len(data)) * 0.5 )
if i == len(data) {
return 0
}
return data[i]
}
func percent90(data []int64)int64{
if len(data) == 0 {
return 0
}
i := int(float64(len(data)) * 0.9 )
if i == len(data) {
return 0
}
return data[i]
}
func percent95(data []int64)int64{
if len(data) == 0 {
return 0
}
i := int(float64(len(data)) * 0.95 )
if i == len(data) {
return 0
}
return data[i]
}
func max(data []int64)int64{
return data[len(data)-1]
}
func statistics(data []int64)(int64, int64, int64){
a := avg(data)
sort.Sort(SortInt64(data))
m := percent50(data)
p9 := percent90(data)
p95 := percent95(data)
ma := max(data)
fmt.Printf("\t avg: %d middle: %dms percent90: %dms percent95: %dms max: %dms\n", a, m, p9, p95, ma)
return a, p95, ma
}
func main(){
var tcase, count int
var host, ak, sk, dir string
flag.IntVar(&tcase, "case", 1, "which test case to use")
flag.IntVar(&count, "count", 100, "total test count")
flag.StringVar(&host, "host", "http://testspeed.xk.design", "access host")
flag.StringVar(&ak, "ak", "", "access key")
flag.StringVar(&sk, "sk", "", "access secret")
flag.StringVar(&dir , "dir", "", "img path")
flag.Parse()
switch tcase {
case 1:
for _, f := range test1 {
test(f, count, host)
testResize(f, count, host)
}
case 2:
for _, f := range test2 {
test(f, count, host)
testResize(f, count, host)
}
case 3:
case3(dir, ak, sk, count, host)
}
}

31
go/uiweb/index.tmpl

@ -0,0 +1,31 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<!-- import CSS -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@4.6.2/dist/css/bootstrap.min.css" integrity="sha384-xOolHFLEh07PJGoPkLv1IbcEPTNtaed2xpHsD9ESMhqIYd0nLMwNLD69Npy4HI+N" crossorigin="anonymous">
<title>{{.title}}</title>
</head>
<body>
<div class="jumbotron jumbotron-fluid" style="height:100vh" >
<div class="container">
<div class="">
<h3 style="text-align: center;">{{.title}}</h3>
</div>
<hr class="my-4" >
{{ range .files }}
<div class="row" style="">
<div style="width:80%;" class="p-3 mb-2 bg-light text-dark"><a href="/ui/{{.}}">{{.}}</a></div>
</div>
{{else}}
<div class="" style="">
<p>这家伙很懒,还没有发布任何信息</p>
</div>
{{end}}
</div>
</div>
<!-- import JavaScript -->
<script src="https://cdn.jsdelivr.net/npm/jquery@3.5.1/dist/jquery.slim.min.js" integrity="sha384-DfXdz2htPH0lsSSs5nCTpuj/zy4C+OGpamoFVy38MVBnE+IbbVYUew+OrCXaRkfj" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@4.6.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-Fy6S3B9q64WdZWQUiU+q4/2Lc9npb8tCaSX9FK7E8HnRr0Jz8D6OP9dO5Vg3Q9ct" crossorigin="anonymous"></script>
</body>
</html>

48
go/uiweb/main.go

File diff suppressed because one or more lines are too long
Loading…
Cancel
Save