Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee4a011a62 | ||
|
|
23f5503a08 | ||
|
|
32a3b2db2e | ||
|
|
f425b0faf6 | ||
|
|
d9cf043cfe | ||
|
|
03e8394bff | ||
|
|
22b4a4afbc | ||
|
|
5a8ee34360 | ||
|
|
1c275e8cfb | ||
|
|
5d138238de | ||
|
|
206f6c8a5d | ||
|
|
8f794be0c1 | ||
|
|
eaf8f06b87 | ||
|
|
5d5312e897 | ||
|
|
08798b6078 | ||
|
|
bf08170f6d | ||
|
|
6d8be979db | ||
|
|
94aefacba4 | ||
|
|
62b971453a | ||
|
|
9dc08d3256 | ||
|
|
dceb314a32 | ||
|
|
3028922835 | ||
|
|
a0a5ba8263 | ||
|
|
99bde764d5 | ||
|
|
c2274379f6 | ||
|
|
204967f05a | ||
|
|
8cd25e4dc2 | ||
|
|
68d4b1898d | ||
|
|
a37f984871 | ||
|
|
4c143310b5 |
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
./build.sh linux amd64
|
||||
./build.sh linux 386
|
||||
#./build.sh linux 386
|
||||
./build.sh linux arm64
|
||||
./build.sh linux mips64
|
||||
./build.sh linux mips64le
|
||||
./build.sh darwin amd64
|
||||
./build.sh darwin arm64
|
||||
#./build.sh linux mips64
|
||||
#./build.sh linux mips64le
|
||||
#./build.sh darwin amd64
|
||||
#./build.sh darwin arm64
|
||||
@@ -6,6 +6,9 @@ if [ -z "$TAG" ]; then
|
||||
TAG="community"
|
||||
fi
|
||||
|
||||
# stop node
|
||||
go run -tags=${TAG} ../cmd/edge-node/main.go stop
|
||||
|
||||
# reference: https://pkg.go.dev/cmd/go/internal/test
|
||||
go clean -testcache
|
||||
go test -timeout 10s -tags="${TAG}" -cover ../...
|
||||
83
go.mod
83
go.mod
@@ -1,23 +1,25 @@
|
||||
module github.com/TeaOSLab/EdgeNode
|
||||
|
||||
go 1.18
|
||||
go 1.21
|
||||
|
||||
replace (
|
||||
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
|
||||
github.com/dchest/captcha => github.com/iwind/captcha v0.0.0-20231130092438-ae985686ed84
|
||||
github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
|
||||
github.com/google/nftables => github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/andybalholm/brotli v1.0.5
|
||||
github.com/aws/aws-sdk-go v1.44.279
|
||||
github.com/baidubce/bce-sdk-go v0.9.153
|
||||
github.com/baidubce/bce-sdk-go v0.9.170
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
|
||||
github.com/fsnotify/fsnotify v1.6.0
|
||||
github.com/cespare/xxhash/v2 v2.2.0
|
||||
github.com/dchest/captcha v0.0.0-00010101000000-000000000000
|
||||
github.com/fsnotify/fsnotify v1.7.0
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/google/gopacket v1.1.19
|
||||
github.com/google/nftables v0.1.0
|
||||
@@ -25,58 +27,81 @@ require (
|
||||
github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
|
||||
github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4
|
||||
github.com/klauspost/compress v1.17.2
|
||||
github.com/iwind/gowebp v0.0.0-20240109104518-489f3429f5c5
|
||||
github.com/klauspost/compress v1.17.7
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mdlayher/netlink v1.7.1
|
||||
github.com/miekg/dns v1.1.43
|
||||
github.com/mssola/useragent v1.0.0
|
||||
github.com/pires/go-proxyproto v0.6.1
|
||||
github.com/qiniu/go-sdk/v7 v7.16.0
|
||||
github.com/quic-go/quic-go v0.39.2
|
||||
github.com/quic-go/quic-go v0.42.0
|
||||
github.com/shirou/gopsutil/v3 v3.22.2
|
||||
github.com/tdewolff/minify/v2 v2.20.19
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.41
|
||||
golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81
|
||||
golang.org/x/image v0.13.0
|
||||
golang.org/x/net v0.17.0
|
||||
golang.org/x/sys v0.13.0
|
||||
google.golang.org/grpc v1.59.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
golang.org/x/net v0.22.0
|
||||
golang.org/x/sys v0.18.0
|
||||
google.golang.org/grpc v1.62.1
|
||||
google.golang.org/protobuf v1.33.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/DataDog/zstd v1.5.5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/clbanning/mxj v1.8.4 // indirect
|
||||
github.com/cockroachdb/errors v1.11.1 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/pebble v1.1.0 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/dgraph-io/ristretto v0.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/getsentry/sentry-go v0.27.0 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/glog v1.2.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/flatbuffers v24.3.7+incompatible // indirect
|
||||
github.com/google/go-cmp v0.6.0 // indirect
|
||||
github.com/google/go-querystring v1.0.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/native v1.0.0 // indirect
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/mdlayher/socket v0.4.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.3 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.16.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/prometheus/client_golang v1.19.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.0 // indirect
|
||||
github.com/prometheus/common v0.51.0 // indirect
|
||||
github.com/prometheus/procfs v0.13.0 // indirect
|
||||
github.com/quic-go/qpack v0.4.0 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
|
||||
github.com/tdewolff/minify/v2 v2.12.7 // indirect
|
||||
github.com/tdewolff/parse/v2 v2.6.6 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
github.com/tdewolff/parse/v2 v2.7.12 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.9 // indirect
|
||||
github.com/tklauser/numcpus v0.3.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.uber.org/mock v0.3.0 // indirect
|
||||
golang.org/x/crypto v0.14.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
|
||||
golang.org/x/mod v0.13.0 // indirect
|
||||
golang.org/x/sync v0.4.0 // indirect
|
||||
golang.org/x/text v0.13.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/mock v0.4.0 // indirect
|
||||
golang.org/x/crypto v0.21.0 // indirect
|
||||
golang.org/x/mod v0.16.0 // indirect
|
||||
golang.org/x/sync v0.6.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.19.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
|
||||
)
|
||||
|
||||
285
go.sum
285
go.sum
@@ -1,34 +1,63 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
|
||||
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/aws/aws-sdk-go v1.44.279 h1:g23dxnYjIiPlQo0gIKNR0zVPsSvo1bj5frWln+5sfhk=
|
||||
github.com/aws/aws-sdk-go v1.44.279/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
|
||||
github.com/baidubce/bce-sdk-go v0.9.153 h1:h5l2EXehe4C4/bdlAPBaULrbnEDgIu5HOYgniN7bjGM=
|
||||
github.com/baidubce/bce-sdk-go v0.9.153/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
|
||||
github.com/baidubce/bce-sdk-go v0.9.170 h1:vAr7COuhu6SEf+8f77DVRji45x7TVZtY5kbu9sX7q8g=
|
||||
github.com/baidubce/bce-sdk-go v0.9.170/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
|
||||
github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
|
||||
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
|
||||
github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4=
|
||||
github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
|
||||
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
|
||||
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
|
||||
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
|
||||
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
|
||||
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
@@ -41,34 +70,64 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
|
||||
github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/flatbuffers v24.3.7+incompatible h1:BxGUkIQnOciBu33bd5BdvqY8Qvo0O/GR4SPhh7x9Ed0=
|
||||
github.com/google/flatbuffers v24.3.7+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
|
||||
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
||||
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0=
|
||||
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q=
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible h1:XRAk4HBDLCYEdPLWtKf5iZhOi7lfx17aY0oSO9+mcg8=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
|
||||
github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d h1:XnTIj781NdSipts60fVqbgZorVAKVSRaA6nqVNfBQ1g=
|
||||
github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
|
||||
github.com/iwind/captcha v0.0.0-20231130092438-ae985686ed84 h1:/RtK8t22a/YFkBWiEwxS+JWcDmxAKsu+r+p00c36K0Q=
|
||||
github.com/iwind/captcha v0.0.0-20231130092438-ae985686ed84/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
|
||||
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
|
||||
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
|
||||
github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4 h1:eyymORsZg0tZ0niyolYF4nao4sdNUI+Ll40s96tKHBY=
|
||||
github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4/go.mod h1:AYyXDhbbD7q9N6rJff2jrE7pGupaiyvtv3YeyIAQLXk=
|
||||
github.com/iwind/gowebp v0.0.0-20240109104518-489f3429f5c5 h1:tA0HEDQJ/FM847wc7kVpSgkTfMF1LervEmd2UZQr3Po=
|
||||
github.com/iwind/gowebp v0.0.0-20240109104518-489f3429f5c5/go.mod h1:AYyXDhbbD7q9N6rJff2jrE7pGupaiyvtv3YeyIAQLXk=
|
||||
github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4 h1:RPAH9Sj9l/20zH5zU5/iJGszfwPq6eLjoiC/n/asulA=
|
||||
github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4/go.mod h1:7OLL+86wZKfBnAJxNxmdcZ0ebbgdp/A28fcagx9oJqA=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
@@ -77,14 +136,21 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk=
|
||||
github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
|
||||
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -92,9 +158,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
|
||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mdlayher/netlink v1.7.1 h1:FdUaT/e33HjEXagwELR8R3/KL1Fq5x3G5jgHLp/BTmg=
|
||||
github.com/mdlayher/netlink v1.7.1/go.mod h1:nKO5CSjE/DJjVhk/TNp6vCE1ktVxEA8VEh8drhZzxsQ=
|
||||
github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw=
|
||||
@@ -103,53 +170,76 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
|
||||
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
|
||||
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
|
||||
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
|
||||
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
|
||||
github.com/mssola/useragent v1.0.0 h1:WRlDpXyxHDNfvZaPEut5Biveq86Ze4o4EMffyMxmH5o=
|
||||
github.com/mssola/useragent v1.0.0/go.mod h1:hz9Cqz4RXusgg1EdI4Al0INR62kP7aPSRNHnpU+b85Y=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
|
||||
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
|
||||
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
|
||||
github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM=
|
||||
github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
|
||||
github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
|
||||
github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
|
||||
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
|
||||
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
|
||||
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
|
||||
github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
|
||||
github.com/prometheus/common v0.51.0 h1:vT5R9NAlW4V6k8Wruk7ikrHaHRsrPbduM/cKTOdQM/k=
|
||||
github.com/prometheus/common v0.51.0/go.mod h1:wHFBCEVWVmHMUpg7pYcOm2QUR/ocQdYSJVQJKnHc3xQ=
|
||||
github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o=
|
||||
github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
|
||||
github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=
|
||||
github.com/qiniu/go-sdk/v7 v7.16.0 h1:Jt4YOMLuaDfgb/KdVg0O1fYLpv5MDkYe/zV+Ri7gWRs=
|
||||
github.com/qiniu/go-sdk/v7 v7.16.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w=
|
||||
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
|
||||
github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
|
||||
github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
|
||||
github.com/quic-go/quic-go v0.39.2 h1:hmwAf8zAHlvan0Y5PXxeeBFZEW17IW99sXLry8I2kjk=
|
||||
github.com/quic-go/quic-go v0.39.2/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
|
||||
github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
|
||||
github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
|
||||
github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tdewolff/minify/v2 v2.12.7 h1:pBzz2tAfz5VghOXiQIsSta6srhmTeinQPjRDHWoumCA=
|
||||
github.com/tdewolff/minify/v2 v2.12.7/go.mod h1:ZRKTheiOGyLSK8hOZWWv+YoJAECzDivNgAlVYDHp/Ws=
|
||||
github.com/tdewolff/parse/v2 v2.6.6 h1:Yld+0CrKUJaCV78DL1G2nk3C9lKrxyRTux5aaK/AkDo=
|
||||
github.com/tdewolff/parse/v2 v2.6.6/go.mod h1:woz0cgbLwFdtbjJu8PIKxhW05KplTFQkOdX78o+Jgrs=
|
||||
github.com/tdewolff/test v1.0.7/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
|
||||
github.com/tdewolff/test v1.0.9 h1:SswqJCmeN4B+9gEAi/5uqT0qpi1y2/2O47V/1hhGZT0=
|
||||
github.com/tdewolff/test v1.0.9/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/tdewolff/minify/v2 v2.20.19 h1:tX0SR0LUrIqGoLjXnkIzRSIbKJ7PaNnSENLD4CyH6Xo=
|
||||
github.com/tdewolff/minify/v2 v2.20.19/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM=
|
||||
github.com/tdewolff/parse/v2 v2.7.12 h1:tgavkHc2ZDEQVKy1oWxwIyh5bP4F5fEh/JmBwPP/3LQ=
|
||||
github.com/tdewolff/parse/v2 v2.7.12/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
|
||||
github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
|
||||
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
|
||||
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.41 h1:iU0Li/Np78H4SBna0ECQoF3mpgi6ImLXU+doGzPFXGc=
|
||||
@@ -159,42 +249,71 @@ github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ
|
||||
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
|
||||
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
|
||||
github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
|
||||
github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
|
||||
go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
|
||||
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
|
||||
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f h1:3CW0unweImhOzd5FmYuRsD4Y4oQFKZIjAnKbjV4WIrw=
|
||||
golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
|
||||
golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc=
|
||||
golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
|
||||
golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
|
||||
golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
|
||||
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
|
||||
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
|
||||
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -204,11 +323,10 @@ golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -218,36 +336,65 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
|
||||
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
|
||||
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
|
||||
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
|
||||
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
|
||||
google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@@ -19,20 +19,21 @@ func currentWeek() int32 {
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Type ItemType `json:"type"`
|
||||
Key string `json:"key"`
|
||||
ExpiredAt int64 `json:"expiredAt"`
|
||||
StaleAt int64 `json:"staleAt"`
|
||||
HeaderSize int64 `json:"headerSize"`
|
||||
BodySize int64 `json:"bodySize"`
|
||||
MetaSize int64 `json:"metaSize"`
|
||||
Host string `json:"host"` // 主机名
|
||||
ServerId int64 `json:"serverId"` // 服务ID
|
||||
Week int32 `json:"week"`
|
||||
Type ItemType `json:"-"`
|
||||
Key string `json:"1,omitempty"`
|
||||
ExpiresAt int64 `json:"2,omitempty"`
|
||||
StaleAt int64 `json:"3,omitempty"`
|
||||
HeaderSize int64 `json:"-"`
|
||||
BodySize int64 `json:"4,omitempty"`
|
||||
MetaSize int64 `json:"-"`
|
||||
Host string `json:"-"` // 主机名
|
||||
ServerId int64 `json:"5,omitempty"` // 服务ID
|
||||
Week int32 `json:"-"`
|
||||
CreatedAt int64 `json:"6,omitempty"`
|
||||
}
|
||||
|
||||
func (this *Item) IsExpired() bool {
|
||||
return this.ExpiredAt < fasttime.Now().Unix()
|
||||
return this.ExpiresAt < fasttime.Now().Unix()
|
||||
}
|
||||
|
||||
func (this *Item) TotalSize() int64 {
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
@@ -13,6 +15,33 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestItem_Marshal(t *testing.T) {
|
||||
{
|
||||
var item = &caches.Item{}
|
||||
data, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(string(data))
|
||||
}
|
||||
|
||||
{
|
||||
var item = &caches.Item{
|
||||
Type: caches.ItemTypeFile,
|
||||
Key: "https://example.com/index.html",
|
||||
ExpiresAt: fasttime.Now().Unix(),
|
||||
HeaderSize: 1 << 10,
|
||||
BodySize: 1 << 20,
|
||||
MetaSize: 256,
|
||||
}
|
||||
data, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(string(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestItems_Memory(t *testing.T) {
|
||||
var stat = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat)
|
||||
|
||||
@@ -20,13 +20,13 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type FileListDB struct {
|
||||
type SQLiteFileListDB struct {
|
||||
dbPath string
|
||||
|
||||
readDB *dbs.DB
|
||||
writeDB *dbs.DB
|
||||
|
||||
hashMap *FileListHashMap
|
||||
hashMap *SQLiteFileListHashMap
|
||||
|
||||
itemsTableName string
|
||||
|
||||
@@ -53,13 +53,13 @@ type FileListDB struct {
|
||||
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
|
||||
}
|
||||
|
||||
func NewFileListDB() *FileListDB {
|
||||
return &FileListDB{
|
||||
hashMap: NewFileListHashMap(),
|
||||
func NewSQLiteFileListDB() *SQLiteFileListDB {
|
||||
return &SQLiteFileListDB{
|
||||
hashMap: NewSQLiteFileListHashMap(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListDB) Open(dbPath string) error {
|
||||
func (this *SQLiteFileListDB) Open(dbPath string) error {
|
||||
this.dbPath = dbPath
|
||||
|
||||
// 动态调整Cache值
|
||||
@@ -119,7 +119,7 @@ func (this *FileListDB) Open(dbPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) Init() error {
|
||||
func (this *SQLiteFileListDB) Init() error {
|
||||
this.itemsTableName = "cacheItems"
|
||||
|
||||
// 创建
|
||||
@@ -184,11 +184,11 @@ func (this *FileListDB) Init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) IsReady() bool {
|
||||
func (this *SQLiteFileListDB) IsReady() bool {
|
||||
return this.isReady
|
||||
}
|
||||
|
||||
func (this *FileListDB) Total() (int64, error) {
|
||||
func (this *SQLiteFileListDB) Total() (int64, error) {
|
||||
// 读取总数量
|
||||
var row = this.readDB.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
|
||||
if row.Err() != nil {
|
||||
@@ -199,14 +199,14 @@ func (this *FileListDB) Total() (int64, error) {
|
||||
return total, err
|
||||
}
|
||||
|
||||
func (this *FileListDB) AddSync(hash string, item *Item) error {
|
||||
func (this *SQLiteFileListDB) AddSync(hash string, item *Item) error {
|
||||
this.hashMap.Add(hash)
|
||||
|
||||
if item.StaleAt == 0 {
|
||||
item.StaleAt = item.ExpiredAt
|
||||
item.StaleAt = item.ExpiresAt
|
||||
}
|
||||
|
||||
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
|
||||
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiresAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
|
||||
if err != nil {
|
||||
return this.WrapError(err)
|
||||
}
|
||||
@@ -214,7 +214,7 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteSync(hash string) error {
|
||||
func (this *SQLiteFileListDB) DeleteSync(hash string) error {
|
||||
this.hashMap.Delete(hash)
|
||||
|
||||
_, err := this.deleteByHashStmt.Exec(hash)
|
||||
@@ -224,7 +224,7 @@ func (this *FileListDB) DeleteSync(hash string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err error) {
|
||||
func (this *SQLiteFileListDB) ListExpiredItems(count int) (hashList []string, err error) {
|
||||
if !this.isReady {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -252,7 +252,7 @@ func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err erro
|
||||
return hashList, nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
func (this *SQLiteFileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
if !this.isReady {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -280,7 +280,7 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
return hashList, nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
|
||||
func (this *SQLiteFileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
|
||||
rows, err := this.selectHashListStmt.Query(lastId)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
@@ -301,12 +301,12 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
|
||||
return
|
||||
}
|
||||
|
||||
func (this *FileListDB) IncreaseHitAsync(hash string) error {
|
||||
func (this *SQLiteFileListDB) IncreaseHitAsync(hash string) error {
|
||||
// do nothing
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanPrefix(prefix string) error {
|
||||
func (this *SQLiteFileListDB) CleanPrefix(prefix string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -327,7 +327,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanMatchKey(key string) error {
|
||||
func (this *SQLiteFileListDB) CleanMatchKey(key string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -372,7 +372,7 @@ func (this *FileListDB) CleanMatchKey(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanMatchPrefix(prefix string) error {
|
||||
func (this *SQLiteFileListDB) CleanMatchPrefix(prefix string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -404,7 +404,7 @@ func (this *FileListDB) CleanMatchPrefix(prefix string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanAll() error {
|
||||
func (this *SQLiteFileListDB) CleanAll() error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -419,7 +419,7 @@ func (this *FileListDB) CleanAll() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) Close() error {
|
||||
func (this *SQLiteFileListDB) Close() error {
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
@@ -477,19 +477,19 @@ func (this *FileListDB) Close() error {
|
||||
return errors.New("close database failed: " + strings.Join(errStrings, ", "))
|
||||
}
|
||||
|
||||
func (this *FileListDB) WrapError(err error) error {
|
||||
func (this *SQLiteFileListDB) WrapError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w (file: %s)", err, this.dbPath)
|
||||
}
|
||||
|
||||
func (this *FileListDB) HashMapIsLoaded() bool {
|
||||
func (this *SQLiteFileListDB) HashMapIsLoaded() bool {
|
||||
return this.hashMapIsLoaded
|
||||
}
|
||||
|
||||
// 初始化
|
||||
func (this *FileListDB) initTables(times int) error {
|
||||
func (this *SQLiteFileListDB) initTables(times int) error {
|
||||
{
|
||||
// expiredAt - 过期时间,用来判断有无过期
|
||||
// staleAt - 过时缓存最大时间,用来清理缓存
|
||||
@@ -553,7 +553,7 @@ ON "` + this.itemsTableName + `" (
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) listOlderItems(count int) (hashList []string, err error) {
|
||||
func (this *SQLiteFileListDB) listOlderItems(count int) (hashList []string, err error) {
|
||||
rows, err := this.listOlderItemsStmt.Query(count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -574,7 +574,7 @@ func (this *FileListDB) listOlderItems(count int) (hashList []string, err error)
|
||||
return hashList, nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) shouldRecover() bool {
|
||||
func (this *SQLiteFileListDB) shouldRecover() bool {
|
||||
result, err := this.writeDB.Query("pragma integrity_check;")
|
||||
if err != nil {
|
||||
logs.Println(result)
|
||||
@@ -592,14 +592,14 @@ func (this *FileListDB) shouldRecover() bool {
|
||||
}
|
||||
|
||||
// 删除数据库文件
|
||||
func (this *FileListDB) deleteDB() {
|
||||
func (this *SQLiteFileListDB) deleteDB() {
|
||||
_ = os.Remove(this.dbPath)
|
||||
_ = os.Remove(this.dbPath + "-shm")
|
||||
_ = os.Remove(this.dbPath + "-wal")
|
||||
}
|
||||
|
||||
// 加载Hash列表
|
||||
func (this *FileListDB) loadHashMap() {
|
||||
func (this *SQLiteFileListDB) loadHashMap() {
|
||||
this.hashMapIsLoaded = false
|
||||
|
||||
err := this.hashMap.Load(this)
|
||||
@@ -18,7 +18,7 @@ func TestFileListDB_ListLFUItems(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -46,7 +46,7 @@ func TestFileListDB_CleanMatchKey(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -78,7 +78,7 @@ func TestFileListDB_CleanMatchPrefix(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -110,7 +110,7 @@ func TestFileListDB_Memory(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -17,8 +17,8 @@ var bigIntPool = sync.Pool{
|
||||
},
|
||||
}
|
||||
|
||||
// FileListHashMap 文件Hash列表
|
||||
type FileListHashMap struct {
|
||||
// SQLiteFileListHashMap 文件Hash列表
|
||||
type SQLiteFileListHashMap struct {
|
||||
m []map[uint64]zero.Zero
|
||||
|
||||
lockers []*sync.RWMutex
|
||||
@@ -27,7 +27,7 @@ type FileListHashMap struct {
|
||||
isReady bool
|
||||
}
|
||||
|
||||
func NewFileListHashMap() *FileListHashMap {
|
||||
func NewSQLiteFileListHashMap() *SQLiteFileListHashMap {
|
||||
var m = make([]map[uint64]zero.Zero, HashMapSharding)
|
||||
var lockers = make([]*sync.RWMutex, HashMapSharding)
|
||||
|
||||
@@ -36,7 +36,7 @@ func NewFileListHashMap() *FileListHashMap {
|
||||
lockers[i] = &sync.RWMutex{}
|
||||
}
|
||||
|
||||
return &FileListHashMap{
|
||||
return &SQLiteFileListHashMap{
|
||||
m: m,
|
||||
lockers: lockers,
|
||||
isAvailable: false,
|
||||
@@ -44,7 +44,7 @@ func NewFileListHashMap() *FileListHashMap {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
func (this *SQLiteFileListHashMap) Load(db *SQLiteFileListDB) error {
|
||||
// 如果系统内存过小,我们不缓存
|
||||
if utils.SystemMemoryGB() < 3 {
|
||||
return nil
|
||||
@@ -75,7 +75,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Add(hash string) {
|
||||
func (this *SQLiteFileListHashMap) Add(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func (this *FileListHashMap) Add(hash string) {
|
||||
this.lockers[index].Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
func (this *SQLiteFileListHashMap) AddHashes(hashes []string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
@@ -100,7 +100,7 @@ func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Delete(hash string) {
|
||||
func (this *SQLiteFileListHashMap) Delete(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
@@ -111,7 +111,7 @@ func (this *FileListHashMap) Delete(hash string) {
|
||||
this.lockers[index].Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Exist(hash string) bool {
|
||||
func (this *SQLiteFileListHashMap) Exist(hash string) bool {
|
||||
if !this.isAvailable {
|
||||
return true
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func (this *FileListHashMap) Exist(hash string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Clean() {
|
||||
func (this *SQLiteFileListHashMap) Clean() {
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
this.lockers[i].Lock()
|
||||
}
|
||||
@@ -143,11 +143,11 @@ func (this *FileListHashMap) Clean() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) IsReady() bool {
|
||||
func (this *SQLiteFileListHashMap) IsReady() bool {
|
||||
return this.isReady
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Len() int {
|
||||
func (this *SQLiteFileListHashMap) Len() int {
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
this.lockers[i].Lock()
|
||||
}
|
||||
@@ -164,15 +164,15 @@ func (this *FileListHashMap) Len() int {
|
||||
return count
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) SetIsAvailable(isAvailable bool) {
|
||||
func (this *SQLiteFileListHashMap) SetIsAvailable(isAvailable bool) {
|
||||
this.isAvailable = isAvailable
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) SetIsReady(isReady bool) {
|
||||
func (this *SQLiteFileListHashMap) SetIsReady(isReady bool) {
|
||||
this.isReady = isReady
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
|
||||
func (this *SQLiteFileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
|
||||
var bigInt = bigIntPool.Get().(*big.Int)
|
||||
bigInt.SetString(hash, 16)
|
||||
hashInt = bigInt.Uint64()
|
||||
@@ -22,7 +22,7 @@ func TestFileListHashMap_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
@@ -91,7 +91,7 @@ func TestFileListHashMap_Load(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -102,7 +102,7 @@ func TestFileListHashMap_Load(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
var before = time.Now()
|
||||
var db = list.GetDB("abc")
|
||||
err = m.Load(db)
|
||||
@@ -121,7 +121,7 @@ func TestFileListHashMap_Load(t *testing.T) {
|
||||
func TestFileListHashMap_Delete(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsReady(true)
|
||||
m.SetIsAvailable(true)
|
||||
m.Add("a")
|
||||
@@ -131,7 +131,7 @@ func TestFileListHashMap_Delete(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Clean(t *testing.T) {
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
m.Clean()
|
||||
m.Add("a")
|
||||
@@ -149,7 +149,7 @@ func Benchmark_BigInt(b *testing.B) {
|
||||
}
|
||||
|
||||
func BenchmarkFileListHashMap_Exist(b *testing.B) {
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
m.SetIsReady(true)
|
||||
|
||||
311
internal/caches/list_file_kv.go
Normal file
311
internal/caches/list_file_kv.go
Normal file
@@ -0,0 +1,311 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const countKVStores = 10
|
||||
|
||||
type KVFileList struct {
|
||||
dir string
|
||||
stores [countKVStores]*KVListFileStore
|
||||
|
||||
onAdd func(item *Item)
|
||||
onRemove func(item *Item)
|
||||
}
|
||||
|
||||
func NewKVFileList(dir string) *KVFileList {
|
||||
dir = strings.TrimSuffix(dir, "/")
|
||||
|
||||
var stores = [countKVStores]*KVListFileStore{}
|
||||
for i := 0; i < countKVStores; i++ {
|
||||
stores[i] = NewKVListFileStore(dir + "/db-" + types.String(i) + ".store")
|
||||
}
|
||||
|
||||
return &KVFileList{
|
||||
dir: dir,
|
||||
stores: stores,
|
||||
}
|
||||
}
|
||||
|
||||
// Init 初始化
|
||||
func (this *KVFileList) Init() error {
|
||||
remotelogs.Println("CACHE", "loading database from '"+this.dir+"' ...")
|
||||
|
||||
var group = goman.NewTaskGroup()
|
||||
var lastErr error
|
||||
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
err := storeCopy.Open()
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("open store '"+storeCopy.Path()+"' failed: %w", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Reset 重置数据
|
||||
func (this *KVFileList) Reset() error {
|
||||
// do nothing
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add 添加内容
|
||||
func (this *KVFileList) Add(hash string, item *Item) error {
|
||||
err := this.getStore(hash).AddItem(hash, item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if this.onAdd != nil {
|
||||
this.onAdd(item)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exist 检查内容是否存在
|
||||
func (this *KVFileList) Exist(hash string) (bool, error) {
|
||||
return this.getStore(hash).ExistItem(hash)
|
||||
}
|
||||
|
||||
// ExistQuick 快速检查内容是否存在
|
||||
func (this *KVFileList) ExistQuick(hash string) (bool, error) {
|
||||
return this.getStore(hash).ExistQuickItem(hash)
|
||||
}
|
||||
|
||||
// CleanPrefix 清除某个前缀的缓存
|
||||
func (this *KVFileList) CleanPrefix(prefix string) error {
|
||||
var group = goman.NewTaskGroup()
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
err := storeCopy.CleanItemsWithPrefix(prefix)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// CleanMatchKey 清除通配符匹配的Key
|
||||
func (this *KVFileList) CleanMatchKey(key string) error {
|
||||
var group = goman.NewTaskGroup()
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
err := storeCopy.CleanItemsWithWildcardKey(key)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// CleanMatchPrefix 清除通配符匹配的前缀
|
||||
func (this *KVFileList) CleanMatchPrefix(prefix string) error {
|
||||
var group = goman.NewTaskGroup()
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
err := storeCopy.CleanItemsWithWildcardPrefix(prefix)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Remove 删除内容
|
||||
func (this *KVFileList) Remove(hash string) error {
|
||||
err := this.getStore(hash).RemoveItem(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if this.onRemove != nil {
|
||||
// when remove file item, no any extra information needed
|
||||
this.onRemove(nil)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Purge 清理过期数据
|
||||
func (this *KVFileList) Purge(count int, callback func(hash string) error) (int, error) {
|
||||
count /= countKVStores
|
||||
if count <= 0 {
|
||||
count = 100
|
||||
}
|
||||
|
||||
var countFound = 0
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
purgeCount, err := store.PurgeItems(count, callback)
|
||||
countFound += purgeCount
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
|
||||
return countFound, lastErr
|
||||
}
|
||||
|
||||
// PurgeLFU 清理LFU数据
|
||||
func (this *KVFileList) PurgeLFU(count int, callback func(hash string) error) error {
|
||||
count /= countKVStores
|
||||
if count <= 0 {
|
||||
count = 100
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
err := store.PurgeLFUItems(count, callback)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// CleanAll 清除所有缓存
|
||||
func (this *KVFileList) CleanAll() error {
|
||||
var group = goman.NewTaskGroup()
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
err := storeCopy.RemoveAllItems()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Stat 统计
|
||||
func (this *KVFileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
var stat = &Stat{}
|
||||
|
||||
var group = goman.NewTaskGroup()
|
||||
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
storeStat, err := storeCopy.StatItems()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
return
|
||||
}
|
||||
|
||||
group.Lock()
|
||||
stat.Size += storeStat.Size
|
||||
stat.ValueSize += storeStat.ValueSize
|
||||
stat.Count += storeStat.Count
|
||||
group.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
group.Wait()
|
||||
|
||||
return stat, lastErr
|
||||
}
|
||||
|
||||
// Count 总数量
|
||||
func (this *KVFileList) Count() (int64, error) {
|
||||
var count int64
|
||||
|
||||
var group = goman.NewTaskGroup()
|
||||
|
||||
var lastErr error
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
countStoreItems, err := storeCopy.CountItems()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
return
|
||||
}
|
||||
|
||||
group.Lock()
|
||||
count += countStoreItems
|
||||
group.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
group.Wait()
|
||||
|
||||
return count, lastErr
|
||||
}
|
||||
|
||||
// OnAdd 添加事件
|
||||
func (this *KVFileList) OnAdd(fn func(item *Item)) {
|
||||
this.onAdd = fn
|
||||
}
|
||||
|
||||
// OnRemove 删除事件
|
||||
func (this *KVFileList) OnRemove(fn func(item *Item)) {
|
||||
this.onRemove = fn
|
||||
}
|
||||
|
||||
// Close 关闭
|
||||
func (this *KVFileList) Close() error {
|
||||
var lastErr error
|
||||
var group = goman.NewTaskGroup()
|
||||
for _, store := range this.stores {
|
||||
var storeCopy = store
|
||||
group.Run(func() {
|
||||
err := storeCopy.Close()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// IncreaseHit 增加点击量
|
||||
func (this *KVFileList) IncreaseHit(hash string) error {
|
||||
// do nothing
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVFileList) TestInspect(t *testing.T) error {
|
||||
for _, store := range this.stores {
|
||||
err := store.TestInspect(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVFileList) getStore(hash string) *KVListFileStore {
|
||||
return this.stores[fnv.HashString(hash)%countKVStores]
|
||||
}
|
||||
66
internal/caches/list_file_kv_objects.go
Normal file
66
internal/caches/list_file_kv_objects.go
Normal file
@@ -0,0 +1,66 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ItemKVEncoder item encoder
|
||||
type ItemKVEncoder[T interface{ *Item }] struct {
|
||||
}
|
||||
|
||||
func NewItemKVEncoder[T interface{ *Item }]() *ItemKVEncoder[T] {
|
||||
return &ItemKVEncoder[T]{}
|
||||
}
|
||||
|
||||
func (this *ItemKVEncoder[T]) Encode(value T) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
|
||||
func (this *ItemKVEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
switch fieldName {
|
||||
case "createdAt":
|
||||
var b = make([]byte, 4)
|
||||
var createdAt = any(value).(*Item).CreatedAt
|
||||
binary.BigEndian.PutUint32(b, uint32(createdAt))
|
||||
return b, nil
|
||||
case "staleAt":
|
||||
var b = make([]byte, 4)
|
||||
var staleAt = any(value).(*Item).StaleAt
|
||||
if staleAt < 0 {
|
||||
staleAt = 0
|
||||
}
|
||||
binary.BigEndian.PutUint32(b, uint32(staleAt))
|
||||
return b, nil
|
||||
case "serverId":
|
||||
var b = make([]byte, 4)
|
||||
var serverId = any(value).(*Item).ServerId
|
||||
if serverId < 0 {
|
||||
serverId = 0
|
||||
}
|
||||
binary.BigEndian.PutUint32(b, uint32(serverId))
|
||||
return b, nil
|
||||
case "key":
|
||||
return []byte(any(value).(*Item).Key), nil
|
||||
case "wildKey":
|
||||
var key = any(value).(*Item).Key
|
||||
var dotIndex = strings.Index(key, ".")
|
||||
if dotIndex > 0 {
|
||||
var slashIndex = strings.LastIndex(key[:dotIndex], "/")
|
||||
if slashIndex > 0 {
|
||||
key = key[:dotIndex][:slashIndex+1] + "*" + key[dotIndex:]
|
||||
}
|
||||
}
|
||||
|
||||
return []byte(key), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (this *ItemKVEncoder[T]) Decode(valueBytes []byte) (value T, err error) {
|
||||
err = json.Unmarshal(valueBytes, &value)
|
||||
return
|
||||
}
|
||||
69
internal/caches/list_file_kv_objects_test.go
Normal file
69
internal/caches/list_file_kv_objects_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestItemKVEncoder_EncodeField(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var encoder = caches.NewItemKVEncoder[*caches.Item]()
|
||||
{
|
||||
key, err := encoder.EncodeField(&caches.Item{
|
||||
Key: "https://example.com/index.html",
|
||||
}, "key")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("key:", string(key))
|
||||
a.IsTrue(string(key) == "https://example.com/index.html")
|
||||
}
|
||||
|
||||
{
|
||||
key, err := encoder.EncodeField(&caches.Item{
|
||||
Key: "",
|
||||
}, "wildKey")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("key:", string(key))
|
||||
a.IsTrue(string(key) == "")
|
||||
}
|
||||
|
||||
{
|
||||
key, err := encoder.EncodeField(&caches.Item{
|
||||
Key: "example.com/index.html",
|
||||
}, "wildKey")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("key:", string(key))
|
||||
a.IsTrue(string(key) == "example.com/index.html")
|
||||
}
|
||||
|
||||
{
|
||||
key, err := encoder.EncodeField(&caches.Item{
|
||||
Key: "https://example.com/index.html",
|
||||
}, "wildKey")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("key:", string(key))
|
||||
a.IsTrue(string(key) == "https://*.com/index.html")
|
||||
}
|
||||
|
||||
{
|
||||
key, err := encoder.EncodeField(&caches.Item{
|
||||
Key: "https://www.example.com/index.html",
|
||||
}, "wildKey")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("key:", string(key))
|
||||
a.IsTrue(string(key) == "https://*.example.com/index.html")
|
||||
}
|
||||
}
|
||||
481
internal/caches/list_file_kv_store.go
Normal file
481
internal/caches/list_file_kv_store.go
Normal file
@@ -0,0 +1,481 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/cockroachdb/pebble"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type KVListFileStore struct {
|
||||
path string
|
||||
rawStore *kvstore.Store
|
||||
|
||||
// tables
|
||||
itemsTable *kvstore.Table[*Item]
|
||||
|
||||
rawIsReady bool
|
||||
}
|
||||
|
||||
func NewKVListFileStore(path string) *KVListFileStore {
|
||||
return &KVListFileStore{
|
||||
path: path,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) Open() error {
|
||||
var reg = regexp.MustCompile(`^(.+)/([\w-]+)(\.store)$`)
|
||||
var matches = reg.FindStringSubmatch(this.path)
|
||||
if len(matches) != 4 {
|
||||
return errors.New("invalid path '" + this.path + "'")
|
||||
}
|
||||
var dir = matches[1]
|
||||
var name = matches[2]
|
||||
|
||||
rawStore, err := kvstore.OpenStoreDir(dir, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.rawStore = rawStore
|
||||
|
||||
db, err := rawStore.NewDB("cache")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
{
|
||||
table, tableErr := kvstore.NewTable[*Item]("items", NewItemKVEncoder[*Item]())
|
||||
if tableErr != nil {
|
||||
return tableErr
|
||||
}
|
||||
|
||||
err = table.AddFields("staleAt", "key", "wildKey", "createdAt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
this.itemsTable = table
|
||||
}
|
||||
|
||||
this.rawIsReady = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) Path() string {
|
||||
return this.path
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) AddItem(hash string, item *Item) error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
if item.ExpiresAt <= currentTime {
|
||||
return nil
|
||||
}
|
||||
if item.CreatedAt <= 0 {
|
||||
item.CreatedAt = currentTime
|
||||
}
|
||||
if item.StaleAt <= 0 {
|
||||
item.StaleAt = item.ExpiresAt + DefaultStaleCacheSeconds
|
||||
}
|
||||
return this.itemsTable.Set(hash, item)
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) ExistItem(hash string) (bool, error) {
|
||||
if !this.isReady() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
item, err := this.itemsTable.Get(hash)
|
||||
if err != nil {
|
||||
if kvstore.IsKeyNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
if item == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return item.ExpiresAt >= fasttime.NewFastTime().Unix(), nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) ExistQuickItem(hash string) (bool, error) {
|
||||
if !this.isReady() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return this.itemsTable.Exist(hash)
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) RemoveItem(hash string) error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return this.itemsTable.Delete(hash)
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) RemoveAllItems() error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return this.itemsTable.Truncate()
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) PurgeItems(count int, callback func(hash string) error) (int, error) {
|
||||
if !this.isReady() {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var countFound int
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
var hashList []string
|
||||
err := this.itemsTable.
|
||||
Query().
|
||||
FieldAsc("staleAt").
|
||||
Limit(count).
|
||||
FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
|
||||
if item.Value == nil {
|
||||
return true, nil
|
||||
}
|
||||
if item.Value.StaleAt < currentTime {
|
||||
countFound++
|
||||
hashList = append(hashList, item.Key)
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// delete items
|
||||
if len(hashList) > 0 {
|
||||
txErr := this.itemsTable.WriteTx(func(tx *kvstore.Tx[*Item]) error {
|
||||
for _, hash := range hashList {
|
||||
deleteErr := tx.Delete(hash)
|
||||
if deleteErr != nil {
|
||||
return deleteErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if txErr != nil {
|
||||
return 0, txErr
|
||||
}
|
||||
|
||||
for _, hash := range hashList {
|
||||
callbackErr := callback(hash)
|
||||
if callbackErr != nil {
|
||||
return 0, callbackErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return countFound, nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) PurgeLFUItems(count int, callback func(hash string) error) error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
var hashList []string
|
||||
err := this.itemsTable.
|
||||
Query().
|
||||
FieldAsc("createdAt").
|
||||
Limit(count).
|
||||
FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
|
||||
if item.Value != nil {
|
||||
hashList = append(hashList, item.Key)
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete items
|
||||
if len(hashList) > 0 {
|
||||
txErr := this.itemsTable.WriteTx(func(tx *kvstore.Tx[*Item]) error {
|
||||
for _, hash := range hashList {
|
||||
deleteErr := tx.Delete(hash)
|
||||
if deleteErr != nil {
|
||||
return deleteErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if txErr != nil {
|
||||
return txErr
|
||||
}
|
||||
|
||||
for _, hash := range hashList {
|
||||
callbackErr := callback(hash)
|
||||
if callbackErr != nil {
|
||||
return callbackErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) CleanItemsWithPrefix(prefix string) error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(prefix) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
|
||||
var fieldOffset []byte
|
||||
const size = 1000
|
||||
for {
|
||||
var count int
|
||||
err := this.itemsTable.
|
||||
Query().
|
||||
FieldPrefix("key", prefix).
|
||||
FieldOffset(fieldOffset).
|
||||
Limit(size).
|
||||
ForUpdate().
|
||||
FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
|
||||
if item.Value == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
count++
|
||||
fieldOffset = item.FieldKey
|
||||
|
||||
if item.Value.CreatedAt >= currentTime {
|
||||
return true, nil
|
||||
}
|
||||
if item.Value.ExpiresAt == 0 {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
item.Value.ExpiresAt = 0
|
||||
item.Value.StaleAt = 0
|
||||
|
||||
setErr := tx.Set(item.Key, item.Value) // TODO improve performance
|
||||
if setErr != nil {
|
||||
return false, setErr
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if count < size {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) CleanItemsWithWildcardPrefix(prefix string) error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(prefix) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
|
||||
var fieldOffset []byte
|
||||
const size = 1000
|
||||
for {
|
||||
var count int
|
||||
err := this.itemsTable.
|
||||
Query().
|
||||
FieldPrefix("wildKey", prefix).
|
||||
FieldOffset(fieldOffset).
|
||||
Limit(size).
|
||||
ForUpdate().
|
||||
FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
|
||||
if item.Value == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
count++
|
||||
fieldOffset = item.FieldKey
|
||||
|
||||
if item.Value.CreatedAt >= currentTime {
|
||||
return true, nil
|
||||
}
|
||||
if item.Value.ExpiresAt == 0 {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
item.Value.ExpiresAt = 0
|
||||
item.Value.StaleAt = 0
|
||||
|
||||
setErr := tx.Set(item.Key, item.Value) // TODO improve performance
|
||||
if setErr != nil {
|
||||
return false, setErr
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if count < size {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) CleanItemsWithWildcardKey(key string) error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(key) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
|
||||
for _, realKey := range []string{key, key + SuffixAll} {
|
||||
var fieldOffset = append(this.itemsTable.FieldKey("wildKey"), '$')
|
||||
fieldOffset = append(fieldOffset, realKey...)
|
||||
const size = 1000
|
||||
|
||||
var wildKey string
|
||||
if !strings.HasSuffix(realKey, SuffixAll) {
|
||||
wildKey = string(append([]byte(realKey), 0, 0))
|
||||
} else {
|
||||
wildKey = realKey
|
||||
}
|
||||
|
||||
for {
|
||||
var count int
|
||||
err := this.itemsTable.
|
||||
Query().
|
||||
FieldPrefix("wildKey", wildKey).
|
||||
FieldOffset(fieldOffset).
|
||||
Limit(size).
|
||||
ForUpdate().
|
||||
FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
|
||||
if item.Value == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
count++
|
||||
fieldOffset = item.FieldKey
|
||||
|
||||
if item.Value.CreatedAt >= currentTime {
|
||||
return true, nil
|
||||
}
|
||||
if item.Value.ExpiresAt == 0 {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
item.Value.ExpiresAt = 0
|
||||
item.Value.StaleAt = 0
|
||||
|
||||
setErr := tx.Set(item.Key, item.Value) // TODO improve performance
|
||||
if setErr != nil {
|
||||
return false, setErr
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if count < size {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) CountItems() (int64, error) {
|
||||
if !this.isReady() {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return this.itemsTable.Count()
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) StatItems() (*Stat, error) {
|
||||
if !this.isReady() {
|
||||
return &Stat{}, nil
|
||||
}
|
||||
|
||||
var stat = &Stat{}
|
||||
|
||||
err := this.itemsTable.
|
||||
Query().
|
||||
FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
|
||||
if item.Value != nil {
|
||||
stat.Size += item.Value.Size()
|
||||
stat.ValueSize += item.Value.BodySize
|
||||
stat.Count++
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
return stat, err
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) TestInspect(t *testing.T) error {
|
||||
if !this.isReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
it, err := this.rawStore.RawDB().NewIter(&pebble.IterOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = it.Close()
|
||||
}()
|
||||
|
||||
for it.First(); it.Valid(); it.Next() {
|
||||
valueBytes, valueErr := it.ValueAndErr()
|
||||
if valueErr != nil {
|
||||
return valueErr
|
||||
}
|
||||
t.Log(string(it.Key()), "=>", string(valueBytes))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) Close() error {
|
||||
if this.rawStore != nil {
|
||||
return this.rawStore.Close()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) isReady() bool {
|
||||
return this.rawIsReady && !this.rawStore.IsClosed()
|
||||
}
|
||||
331
internal/caches/list_file_kv_test.go
Normal file
331
internal/caches/list_file_kv_test.go
Normal file
@@ -0,0 +1,331 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var testingKVList *caches.KVFileList
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
m.Run()
|
||||
|
||||
if testingKVList != nil {
|
||||
_ = testingKVList.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func testOpenKVFileList(t *testing.T) *caches.KVFileList {
|
||||
var list = caches.NewKVFileList(Tea.Root + "/data/stores/cache-stores")
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testingKVList = list
|
||||
return list
|
||||
}
|
||||
|
||||
func TestNewKVFileList(t *testing.T) {
|
||||
_ = testOpenKVFileList(t)
|
||||
}
|
||||
|
||||
func TestKVFileList_Add(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
|
||||
err := list.Add(stringutil.Md5("123456"), &caches.Item{
|
||||
Type: caches.ItemTypeFile,
|
||||
Key: "https://example.com/index.html",
|
||||
ExpiresAt: time.Now().Unix() + 60,
|
||||
StaleAt: 0,
|
||||
HeaderSize: 0,
|
||||
BodySize: 4096,
|
||||
MetaSize: 0,
|
||||
Host: "",
|
||||
ServerId: 1,
|
||||
Week: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_Add_Many(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var list = testOpenKVFileList(t)
|
||||
|
||||
const start = 0
|
||||
const count = 1_000_000
|
||||
const concurrent = 100
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", fmt.Sprintf("%.2fs", costSeconds), "qps:", fmt.Sprintf("%.2fK/s", float64(count)/1000/costSeconds))
|
||||
}()
|
||||
|
||||
var wg = &sync.WaitGroup{}
|
||||
wg.Add(concurrent)
|
||||
for c := 0; c < concurrent; c++ {
|
||||
go func(c int) {
|
||||
defer wg.Done()
|
||||
|
||||
var segmentStart = start + count/concurrent*c
|
||||
for i := segmentStart; i < segmentStart+count/concurrent; i++ {
|
||||
err := list.Add(stringutil.Md5(strconv.Itoa(i)), &caches.Item{
|
||||
Type: caches.ItemTypeFile,
|
||||
Key: "https://www.example.com/index.html" + strconv.Itoa(i),
|
||||
ExpiresAt: time.Now().Unix() + 60,
|
||||
StaleAt: 0,
|
||||
HeaderSize: 0,
|
||||
BodySize: int64(rand.Int() % 1_000_000),
|
||||
MetaSize: 0,
|
||||
Host: "",
|
||||
ServerId: 1,
|
||||
Week: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Log(err)
|
||||
}
|
||||
}
|
||||
}(c)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestKVFileList_Add_Many_Suffix(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var list = testOpenKVFileList(t)
|
||||
|
||||
const start = 0
|
||||
const count = 1000
|
||||
const concurrent = 100
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", fmt.Sprintf("%.2fs", costSeconds), "qps:", fmt.Sprintf("%.2fK/s", float64(count)/1000/costSeconds))
|
||||
}()
|
||||
|
||||
var wg = &sync.WaitGroup{}
|
||||
wg.Add(concurrent)
|
||||
for c := 0; c < concurrent; c++ {
|
||||
go func(c int) {
|
||||
defer wg.Done()
|
||||
|
||||
var segmentStart = start + count/concurrent*c
|
||||
for i := segmentStart; i < segmentStart+count/concurrent; i++ {
|
||||
err := list.Add(stringutil.Md5(strconv.Itoa(i)+caches.SuffixAll), &caches.Item{
|
||||
Type: caches.ItemTypeFile,
|
||||
Key: "https://www.example.com/index.html" + strconv.Itoa(i) + caches.SuffixAll + "zip",
|
||||
ExpiresAt: time.Now().Unix() + 60,
|
||||
StaleAt: 0,
|
||||
HeaderSize: 0,
|
||||
BodySize: int64(rand.Int() % 1_000_000),
|
||||
MetaSize: 0,
|
||||
Host: "",
|
||||
ServerId: 1,
|
||||
Week: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Log(err)
|
||||
}
|
||||
}
|
||||
}(c)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestKVFileList_Exist(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
for _, hash := range []string{
|
||||
stringutil.Md5("123456"),
|
||||
stringutil.Md5("654321"),
|
||||
} {
|
||||
b, err := list.Exist(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(hash, "=>", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_ExistQuick(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
for _, hash := range []string{
|
||||
stringutil.Md5("123456"),
|
||||
stringutil.Md5("654321"),
|
||||
} {
|
||||
b, err := list.ExistQuick(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(hash, "=>", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_Remove(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
for _, hash := range []string{
|
||||
stringutil.Md5("123456"),
|
||||
stringutil.Md5("654321"),
|
||||
} {
|
||||
err := list.Remove(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_CleanAll(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
err := list.CleanAll()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_Inspect(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var list = testOpenKVFileList(t)
|
||||
err := list.TestInspect(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_Purge(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
var before = time.Now()
|
||||
count, err := list.Purge(4_000, func(hash string) error {
|
||||
//t.Log("hash:", hash)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "count:", count)
|
||||
}
|
||||
|
||||
func TestKVFileList_PurgeLFU(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
var before = time.Now()
|
||||
err := list.PurgeLFU(20000, func(hash string) error {
|
||||
t.Log("hash:", hash)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000))
|
||||
}
|
||||
|
||||
func TestKVFileList_Count(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
var before = time.Now()
|
||||
count, err := list.Count()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "count:", count)
|
||||
}
|
||||
|
||||
func TestKVFileList_Stat(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
var before = time.Now()
|
||||
stat, err := list.Stat(func(hash string) bool {
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "stat:", fmt.Sprintf("%+v", stat))
|
||||
}
|
||||
|
||||
func TestKVFileList_CleanPrefix(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
var before = time.Now()
|
||||
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000))
|
||||
}()
|
||||
|
||||
err := list.CleanPrefix("https://www.example.com/index.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_CleanMatchPrefix(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
var before = time.Now()
|
||||
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000))
|
||||
}()
|
||||
|
||||
err := list.CleanMatchPrefix("https://*.example.com/index.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVFileList_CleanMatchKey(t *testing.T) {
|
||||
var list = testOpenKVFileList(t)
|
||||
var before = time.Now()
|
||||
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000))
|
||||
}()
|
||||
|
||||
err := list.CleanMatchKey("https://*.example.com/index.html123")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkKVFileList_Exist(b *testing.B) {
|
||||
var list = caches.NewKVFileList(Tea.Root + "/data/stores/cache-stores")
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
}()
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_, existErr := list.Exist(stringutil.Md5(strconv.Itoa(rand.Int() % 2_000_000)))
|
||||
if existErr != nil {
|
||||
b.Fatal(existErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -21,10 +21,10 @@ import (
|
||||
|
||||
const CountFileDB = 20
|
||||
|
||||
// FileList 文件缓存列表管理
|
||||
type FileList struct {
|
||||
// SQLiteFileList 文件缓存列表管理
|
||||
type SQLiteFileList struct {
|
||||
dir string
|
||||
dbList [CountFileDB]*FileListDB
|
||||
dbList [CountFileDB]*SQLiteFileListDB
|
||||
|
||||
onAdd func(item *Item)
|
||||
onRemove func(item *Item)
|
||||
@@ -35,18 +35,18 @@ type FileList struct {
|
||||
oldDir string
|
||||
}
|
||||
|
||||
func NewFileList(dir string) ListInterface {
|
||||
return &FileList{
|
||||
func NewSQLiteFileList(dir string) ListInterface {
|
||||
return &SQLiteFileList{
|
||||
dir: dir,
|
||||
memoryCache: ttlcache.NewCache[zero.Zero](),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileList) SetOldDir(oldDir string) {
|
||||
func (this *SQLiteFileList) SetOldDir(oldDir string) {
|
||||
this.oldDir = oldDir
|
||||
}
|
||||
|
||||
func (this *FileList) Init() error {
|
||||
func (this *SQLiteFileList) Init() error {
|
||||
// 检查目录是否存在
|
||||
_, err := os.Stat(this.dir)
|
||||
if err != nil {
|
||||
@@ -73,7 +73,7 @@ func (this *FileList) Init() error {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
|
||||
var db = NewFileListDB()
|
||||
var db = NewSQLiteFileListDB()
|
||||
dbErr := db.Open(dir + "/db-" + types.String(i) + ".db")
|
||||
if dbErr != nil {
|
||||
lastErr = dbErr
|
||||
@@ -105,12 +105,12 @@ func (this *FileList) Init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Reset() error {
|
||||
func (this *SQLiteFileList) Reset() error {
|
||||
// 不做任何事情
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Add(hash string, item *Item) error {
|
||||
func (this *SQLiteFileList) Add(hash string, item *Item) error {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -122,7 +122,7 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiredAt))
|
||||
this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiresAt))
|
||||
|
||||
if this.onAdd != nil {
|
||||
this.onAdd(item)
|
||||
@@ -130,7 +130,7 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Exist(hash string) (bool, error) {
|
||||
func (this *SQLiteFileList) Exist(hash string) (bool, error) {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -160,11 +160,16 @@ func (this *FileList) Exist(hash string) (bool, error) {
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
if expiredAt < fasttime.Now().Unix() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(expiredAt))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
|
||||
func (this *SQLiteFileList) ExistQuick(hash string) (isReady bool, found bool) {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() || !db.HashMapIsLoaded() {
|
||||
@@ -177,7 +182,7 @@ func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
|
||||
}
|
||||
|
||||
// CleanPrefix 清理某个前缀的缓存数据
|
||||
func (this *FileList) CleanPrefix(prefix string) error {
|
||||
func (this *SQLiteFileList) CleanPrefix(prefix string) error {
|
||||
if len(prefix) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -197,7 +202,7 @@ func (this *FileList) CleanPrefix(prefix string) error {
|
||||
}
|
||||
|
||||
// CleanMatchKey 清理通配符匹配的缓存数据,类似于 https://*.example.com/hello
|
||||
func (this *FileList) CleanMatchKey(key string) error {
|
||||
func (this *SQLiteFileList) CleanMatchKey(key string) error {
|
||||
if len(key) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -217,7 +222,7 @@ func (this *FileList) CleanMatchKey(key string) error {
|
||||
}
|
||||
|
||||
// CleanMatchPrefix 清理通配符匹配的缓存数据,类似于 https://*.example.com/prefix/
|
||||
func (this *FileList) CleanMatchPrefix(prefix string) error {
|
||||
func (this *SQLiteFileList) CleanMatchPrefix(prefix string) error {
|
||||
if len(prefix) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -236,7 +241,7 @@ func (this *FileList) CleanMatchPrefix(prefix string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Remove(hash string) error {
|
||||
func (this *SQLiteFileList) Remove(hash string) error {
|
||||
_, err := this.remove(hash, false)
|
||||
return err
|
||||
}
|
||||
@@ -244,7 +249,7 @@ func (this *FileList) Remove(hash string) error {
|
||||
// Purge 清理过期的缓存
|
||||
// count 每次遍历的最大数量,控制此数字可以保证每次清理的时候不用花太多时间
|
||||
// callback 每次发现过期key的调用
|
||||
func (this *FileList) Purge(count int, callback func(hash string) error) (int, error) {
|
||||
func (this *SQLiteFileList) Purge(count int, callback func(hash string) error) (int, error) {
|
||||
count /= CountFileDB
|
||||
if count <= 0 {
|
||||
count = 100
|
||||
@@ -285,7 +290,7 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
|
||||
return countFound, nil
|
||||
}
|
||||
|
||||
func (this *FileList) PurgeLFU(count int, callback func(hash string) error) error {
|
||||
func (this *SQLiteFileList) PurgeLFU(count int, callback func(hash string) error) error {
|
||||
count /= CountFileDB
|
||||
if count <= 0 {
|
||||
count = 100
|
||||
@@ -322,7 +327,7 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) CleanAll() error {
|
||||
func (this *SQLiteFileList) CleanAll() error {
|
||||
defer this.memoryCache.Clean()
|
||||
|
||||
for _, db := range this.dbList {
|
||||
@@ -335,7 +340,7 @@ func (this *FileList) CleanAll() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
func (this *SQLiteFileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
var result = &Stat{}
|
||||
|
||||
for _, db := range this.dbList {
|
||||
@@ -365,7 +370,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
|
||||
// Count 总数量
|
||||
// 常用的方法,所以避免直接查询数据库
|
||||
func (this *FileList) Count() (int64, error) {
|
||||
func (this *SQLiteFileList) Count() (int64, error) {
|
||||
var total int64
|
||||
for _, db := range this.dbList {
|
||||
count, err := db.Total()
|
||||
@@ -378,7 +383,7 @@ func (this *FileList) Count() (int64, error) {
|
||||
}
|
||||
|
||||
// IncreaseHit 增加点击量
|
||||
func (this *FileList) IncreaseHit(hash string) error {
|
||||
func (this *SQLiteFileList) IncreaseHit(hash string) error {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -389,36 +394,41 @@ func (this *FileList) IncreaseHit(hash string) error {
|
||||
}
|
||||
|
||||
// OnAdd 添加事件
|
||||
func (this *FileList) OnAdd(f func(item *Item)) {
|
||||
func (this *SQLiteFileList) OnAdd(f func(item *Item)) {
|
||||
this.onAdd = f
|
||||
}
|
||||
|
||||
// OnRemove 删除事件
|
||||
func (this *FileList) OnRemove(f func(item *Item)) {
|
||||
func (this *SQLiteFileList) OnRemove(f func(item *Item)) {
|
||||
this.onRemove = f
|
||||
}
|
||||
|
||||
func (this *FileList) Close() error {
|
||||
func (this *SQLiteFileList) Close() error {
|
||||
this.memoryCache.Destroy()
|
||||
|
||||
var group = goman.NewTaskGroup()
|
||||
for _, db := range this.dbList {
|
||||
if db != nil {
|
||||
_ = db.Close()
|
||||
}
|
||||
var dbCopy = db
|
||||
group.Run(func() {
|
||||
if dbCopy != nil {
|
||||
_ = dbCopy.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) GetDBIndex(hash string) uint64 {
|
||||
func (this *SQLiteFileList) GetDBIndex(hash string) uint64 {
|
||||
return fnv.HashString(hash) % CountFileDB
|
||||
}
|
||||
|
||||
func (this *FileList) GetDB(hash string) *FileListDB {
|
||||
func (this *SQLiteFileList) GetDB(hash string) *SQLiteFileListDB {
|
||||
return this.dbList[fnv.HashString(hash)%CountFileDB]
|
||||
}
|
||||
|
||||
func (this *FileList) HashMapIsLoaded() bool {
|
||||
func (this *SQLiteFileList) HashMapIsLoaded() bool {
|
||||
for _, db := range this.dbList {
|
||||
if !db.HashMapIsLoaded() {
|
||||
return false
|
||||
@@ -427,7 +437,7 @@ func (this *FileList) HashMapIsLoaded() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
|
||||
func (this *SQLiteFileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -459,14 +469,14 @@ func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err er
|
||||
}
|
||||
|
||||
// 升级老版本数据库
|
||||
func (this *FileList) upgradeOldDB() {
|
||||
func (this *SQLiteFileList) upgradeOldDB() {
|
||||
if len(this.oldDir) == 0 {
|
||||
return
|
||||
}
|
||||
_ = this.UpgradeV3(this.oldDir, false)
|
||||
}
|
||||
|
||||
func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
func (this *SQLiteFileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
// index.db
|
||||
var indexDBPath = oldDir + "/index.db"
|
||||
_, err := os.Stat(indexDBPath)
|
||||
@@ -538,7 +548,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
err = this.Add(hash, &Item{
|
||||
Type: ItemTypeFile,
|
||||
Key: key,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
StaleAt: staleAt,
|
||||
HeaderSize: headerSize,
|
||||
BodySize: bodySize,
|
||||
@@ -565,7 +575,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
|
||||
func (this *SQLiteFileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
|
||||
var maxTimestamp = fasttime.Now().Unix() + 3600
|
||||
if expiresAt > maxTimestamp {
|
||||
return maxTimestamp
|
||||
@@ -21,7 +21,7 @@ func TestFileList_Init(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -42,7 +42,7 @@ func TestFileList_Add(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -61,7 +61,7 @@ func TestFileList_Add(t *testing.T) {
|
||||
t.Log("db index:", list.GetDBIndex(hash))
|
||||
err = list.Add(hash, &caches.Item{
|
||||
Key: "123456",
|
||||
ExpiredAt: time.Now().Unix() + 1,
|
||||
ExpiresAt: time.Now().Unix() + 1,
|
||||
HeaderSize: 1,
|
||||
MetaSize: 2,
|
||||
BodySize: 3,
|
||||
@@ -82,7 +82,7 @@ func TestFileList_Add_Many(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -94,11 +94,13 @@ func TestFileList_Add_Many(t *testing.T) {
|
||||
}
|
||||
|
||||
var before = time.Now()
|
||||
for i := 0; i < 10_000_000; i++ {
|
||||
const offset = 0
|
||||
const count = 1_000_000
|
||||
for i := offset; i < offset+count; i++ {
|
||||
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
|
||||
_ = list.Add(stringutil.Md5(u), &caches.Item{
|
||||
Key: u,
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1,
|
||||
MetaSize: 2,
|
||||
BodySize: 3,
|
||||
@@ -119,7 +121,7 @@ func TestFileList_Exist(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
}()
|
||||
@@ -162,7 +164,7 @@ func TestFileList_Exist_Many_DB(t *testing.T) {
|
||||
// 测试在多个数据库下的性能
|
||||
var listSlice = []caches.ListInterface{}
|
||||
for i := 1; i <= 10; i++ {
|
||||
var list = caches.NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -222,7 +224,7 @@ func TestFileList_CleanPrefix(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -246,7 +248,7 @@ func TestFileList_Remove(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
}()
|
||||
@@ -274,7 +276,7 @@ func TestFileList_Purge(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -302,7 +304,7 @@ func TestFileList_PurgeLFU(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -330,7 +332,7 @@ func TestFileList_Stat(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -353,7 +355,7 @@ func TestFileList_Count(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -377,7 +379,7 @@ func TestFileList_CleanAll(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -400,7 +402,7 @@ func TestFileList_UpgradeV3(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p43").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p43").(*caches.SQLiteFileList)
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -428,7 +430,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -115,7 +115,7 @@ func (this *MemoryList) CleanPrefix(prefix string) error {
|
||||
for _, itemMap := range this.itemMaps {
|
||||
for _, item := range itemMap {
|
||||
if strings.HasPrefix(item.Key, prefix) {
|
||||
item.ExpiredAt = 0
|
||||
item.ExpiresAt = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -153,7 +153,7 @@ func (this *MemoryList) CleanMatchKey(key string) error {
|
||||
if configutils.MatchDomain(host, item.Host) {
|
||||
var itemRequestURI = item.RequestURI()
|
||||
if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) {
|
||||
item.ExpiredAt = 0
|
||||
item.ExpiresAt = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -189,7 +189,7 @@ func (this *MemoryList) CleanMatchPrefix(prefix string) error {
|
||||
if configutils.MatchDomain(host, item.Host) {
|
||||
var itemRequestURI = item.RequestURI()
|
||||
if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) {
|
||||
item.ExpiredAt = 0
|
||||
item.ExpiresAt = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,17 +21,17 @@ func TestMemoryList_Add(t *testing.T) {
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &caches.Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("b", &caches.Item{
|
||||
Key: "b1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("123456", &caches.Item{
|
||||
Key: "c1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
t.Log(list.Prefixes())
|
||||
@@ -44,12 +44,12 @@ func TestMemoryList_Remove(t *testing.T) {
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &caches.Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("b", &caches.Item{
|
||||
Key: "b1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Remove("b")
|
||||
@@ -62,22 +62,22 @@ func TestMemoryList_Purge(t *testing.T) {
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &caches.Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("b", &caches.Item{
|
||||
Key: "b1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("c", &caches.Item{
|
||||
Key: "c1",
|
||||
ExpiredAt: time.Now().Unix() - 3600,
|
||||
ExpiresAt: time.Now().Unix() - 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("d", &caches.Item{
|
||||
Key: "d1",
|
||||
ExpiredAt: time.Now().Unix() - 2,
|
||||
ExpiresAt: time.Now().Unix() - 2,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_, _ = list.Purge(100, func(hash string) error {
|
||||
@@ -109,7 +109,7 @@ func TestMemoryList_Purge_Large_List(t *testing.T) {
|
||||
for i := 0; i < count; i++ {
|
||||
_ = list.Add("a"+strconv.Itoa(i), &caches.Item{
|
||||
Key: "a" + strconv.Itoa(i),
|
||||
ExpiredAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)),
|
||||
ExpiresAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)),
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
}
|
||||
@@ -124,22 +124,22 @@ func TestMemoryList_Stat(t *testing.T) {
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &caches.Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("b", &caches.Item{
|
||||
Key: "b1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("c", &caches.Item{
|
||||
Key: "c1",
|
||||
ExpiredAt: time.Now().Unix(),
|
||||
ExpiresAt: time.Now().Unix(),
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Add("d", &caches.Item{
|
||||
Key: "d1",
|
||||
ExpiredAt: time.Now().Unix() - 2,
|
||||
ExpiresAt: time.Now().Unix() - 2,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
result, _ := list.Stat(func(hash string) bool {
|
||||
@@ -161,7 +161,7 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
|
||||
key := "https://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
|
||||
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
|
||||
Key: key,
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
BodySize: 0,
|
||||
HeaderSize: 0,
|
||||
})
|
||||
@@ -278,7 +278,7 @@ func TestMemoryList_GC(t *testing.T) {
|
||||
key := "https://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
|
||||
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
|
||||
Key: key,
|
||||
ExpiredAt: 0,
|
||||
ExpiresAt: 0,
|
||||
BodySize: 0,
|
||||
HeaderSize: 0,
|
||||
})
|
||||
@@ -308,7 +308,7 @@ func BenchmarkMemoryList(b *testing.B) {
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
_ = list.Add(stringutil.Md5(types.String(i)), &caches.Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
ExpiresAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -258,12 +258,25 @@ func (this *FileStorage) Init() error {
|
||||
return errors.New("[CACHE]cache storage dir can not be empty")
|
||||
}
|
||||
|
||||
var list = NewFileList(dir + "/p" + types.String(this.policy.Id) + "/.indexes")
|
||||
err = list.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
// read list
|
||||
var list ListInterface
|
||||
var sqliteIndexesDir = dir + "/p" + types.String(this.policy.Id) + "/.indexes"
|
||||
_, sqliteIndexesDirErr := os.Stat(sqliteIndexesDir)
|
||||
if sqliteIndexesDirErr == nil || !teaconst.EnableKVCacheStore {
|
||||
list = NewSQLiteFileList(sqliteIndexesDir)
|
||||
err = list.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
list.(*SQLiteFileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
|
||||
} else {
|
||||
list = NewKVFileList(dir + "/p" + types.String(this.policy.Id) + "/.stores")
|
||||
err = list.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
list.(*FileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
|
||||
|
||||
this.list = list
|
||||
|
||||
// 检查目录是否存在
|
||||
@@ -1294,7 +1307,7 @@ func (this *FileStorage) hotLoop() {
|
||||
Type: writer.ItemType(),
|
||||
Key: item.Key,
|
||||
Host: ParseHost(item.Key),
|
||||
ExpiredAt: expiresAt,
|
||||
ExpiresAt: expiresAt,
|
||||
HeaderSize: writer.HeaderSize(),
|
||||
BodySize: writer.BodySize(),
|
||||
})
|
||||
@@ -1608,7 +1621,8 @@ func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
|
||||
// ScanGarbageCaches 清理目录中“失联”的缓存文件
|
||||
// “失联”为不在HashMap中的文件
|
||||
func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error) error {
|
||||
if !this.list.(*FileList).HashMapIsLoaded() {
|
||||
_, isSQLite := this.list.(*SQLiteFileList)
|
||||
if isSQLite && !this.list.(*SQLiteFileList).HashMapIsLoaded() {
|
||||
return errors.New("cache list is loading")
|
||||
}
|
||||
|
||||
@@ -1641,8 +1655,8 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
continue
|
||||
}
|
||||
|
||||
dir2Matches, err := filepath.Glob(dir1 + "/*")
|
||||
if err != nil {
|
||||
dir2Matches, globErr := filepath.Glob(dir1 + "/*")
|
||||
if globErr != nil {
|
||||
// ignore error
|
||||
continue
|
||||
}
|
||||
@@ -1665,8 +1679,8 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
}
|
||||
}
|
||||
|
||||
fileMatches, err := filepath.Glob(dir2 + "/*.cache")
|
||||
if err != nil {
|
||||
fileMatches, globDir2Err := filepath.Glob(dir2 + "/*.cache")
|
||||
if globDir2Err != nil {
|
||||
// ignore error
|
||||
continue
|
||||
}
|
||||
@@ -1678,7 +1692,19 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
continue
|
||||
}
|
||||
|
||||
isReady, found := this.list.(*FileList).ExistQuick(hash)
|
||||
var isReady, found bool
|
||||
switch rawList := this.list.(type) {
|
||||
case *SQLiteFileList:
|
||||
isReady, found = rawList.ExistQuick(hash)
|
||||
case *KVFileList:
|
||||
isReady = true
|
||||
var checkErr error
|
||||
found, checkErr = rawList.ExistQuick(hash)
|
||||
if checkErr != nil {
|
||||
return checkErr
|
||||
}
|
||||
}
|
||||
|
||||
if !isReady {
|
||||
continue
|
||||
}
|
||||
@@ -1688,8 +1714,8 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
}
|
||||
|
||||
// 检查文件正在被写入
|
||||
stat, err := os.Stat(file)
|
||||
if err != nil {
|
||||
stat, statErr := os.Stat(file)
|
||||
if statErr != nil {
|
||||
continue
|
||||
}
|
||||
if fasttime.Now().Unix()-stat.ModTime().Unix() < 300 /** 5 minutes **/ {
|
||||
@@ -1698,9 +1724,9 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
|
||||
if fileCallback != nil {
|
||||
countFound++
|
||||
err = fileCallback(file)
|
||||
if err != nil {
|
||||
return err
|
||||
callbackErr := fileCallback(file)
|
||||
if callbackErr != nil {
|
||||
return callbackErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ func TestFileStorage_Init(t *testing.T) {
|
||||
|
||||
time.Sleep(2 * time.Second)
|
||||
storage.purgeLoop()
|
||||
t.Log(storage.list.(*FileList).Stat(func(hash string) bool {
|
||||
t.Log(storage.list.(*SQLiteFileList).Stat(func(hash string) bool {
|
||||
return true
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -578,7 +578,7 @@ func (this *MemoryStorage) flushItem(key string) {
|
||||
Type: writer.ItemType(),
|
||||
Key: key,
|
||||
Host: ParseHost(key),
|
||||
ExpiredAt: item.ExpiresAt,
|
||||
ExpiresAt: item.ExpiresAt,
|
||||
HeaderSize: writer.HeaderSize(),
|
||||
BodySize: writer.BodySize(),
|
||||
})
|
||||
|
||||
@@ -157,7 +157,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
|
||||
storage.AddToList(&Item{
|
||||
Key: "abc",
|
||||
BodySize: 5,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
})
|
||||
}
|
||||
{
|
||||
@@ -174,7 +174,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
|
||||
storage.AddToList(&Item{
|
||||
Key: "abc1",
|
||||
BodySize: 5,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
})
|
||||
}
|
||||
stat, err := storage.Stat()
|
||||
@@ -201,7 +201,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
|
||||
storage.AddToList(&Item{
|
||||
Key: "abc",
|
||||
BodySize: 5,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
})
|
||||
}
|
||||
{
|
||||
@@ -217,7 +217,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
|
||||
storage.AddToList(&Item{
|
||||
Key: "abc1",
|
||||
BodySize: 5,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
})
|
||||
}
|
||||
err := storage.CleanAll()
|
||||
@@ -244,7 +244,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
|
||||
storage.AddToList(&Item{
|
||||
Key: "abc",
|
||||
BodySize: 5,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
})
|
||||
}
|
||||
{
|
||||
@@ -260,7 +260,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
|
||||
storage.AddToList(&Item{
|
||||
Key: "abc1",
|
||||
BodySize: 5,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
})
|
||||
}
|
||||
err := storage.Purge([]string{"abc", "abc1"}, "")
|
||||
@@ -299,7 +299,7 @@ func TestMemoryStorage_Expire(t *testing.T) {
|
||||
storage.AddToList(&Item{
|
||||
Key: key,
|
||||
BodySize: 5,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
})
|
||||
}
|
||||
time.Sleep(70 * time.Second)
|
||||
|
||||
@@ -48,6 +48,10 @@ func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64,
|
||||
SharedFragmentMemoryPool.IncreaseNew()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if expectedBodySize > 0 {
|
||||
valueItem.BodyValue = make([]byte, 0, expectedBodySize)
|
||||
}
|
||||
}
|
||||
var w = &MemoryWriter{
|
||||
storage: memoryStorage,
|
||||
|
||||
143
internal/caches/writer_memory_test.go
Normal file
143
internal/caches/writer_memory_test.go
Normal file
@@ -0,0 +1,143 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewMemoryWriter(t *testing.T) {
|
||||
var storage = caches.NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
|
||||
Id: 0,
|
||||
IsOn: false,
|
||||
Name: "",
|
||||
Description: "",
|
||||
Capacity: &shared.SizeCapacity{
|
||||
Count: 8,
|
||||
Unit: shared.SizeCapacityUnitGB,
|
||||
},
|
||||
}, nil)
|
||||
err := storage.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const size = 1 << 20
|
||||
const chunkSize = 16 << 10
|
||||
var data = bytes.Repeat([]byte{'A'}, chunkSize)
|
||||
|
||||
var before = time.Now()
|
||||
|
||||
var writer = caches.NewMemoryWriter(storage, "a", time.Now().Unix()+3600, 200, false, size, 1<<30, func(valueItem *caches.MemoryItem) {
|
||||
t.Log(len(valueItem.BodyValue), "bytes")
|
||||
})
|
||||
|
||||
for i := 0; i < size/chunkSize; i++ {
|
||||
_, err = writer.Write(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
|
||||
func BenchmarkMemoryWriter_Capacity(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
var storage = caches.NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
|
||||
Id: 0,
|
||||
IsOn: false,
|
||||
Name: "",
|
||||
Description: "",
|
||||
Capacity: &shared.SizeCapacity{
|
||||
Count: 8,
|
||||
Unit: shared.SizeCapacityUnitGB,
|
||||
},
|
||||
}, nil)
|
||||
initErr := storage.Init()
|
||||
if initErr != nil {
|
||||
b.Fatal(initErr)
|
||||
}
|
||||
|
||||
const size = 1 << 20
|
||||
const chunkSize = 16 << 10
|
||||
var data = bytes.Repeat([]byte{'A'}, chunkSize)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
var writer = caches.NewMemoryWriter(storage, "a"+strconv.Itoa(rand.Int()), time.Now().Unix()+3600, 200, false, size, 1<<30, func(valueItem *caches.MemoryItem) {
|
||||
})
|
||||
|
||||
for i := 0; i < size/chunkSize; i++ {
|
||||
_, err := writer.Write(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkMemoryWriter_Capacity_Disabled(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
var storage = caches.NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
|
||||
Id: 0,
|
||||
IsOn: false,
|
||||
Name: "",
|
||||
Description: "",
|
||||
Capacity: &shared.SizeCapacity{
|
||||
Count: 8,
|
||||
Unit: shared.SizeCapacityUnitGB,
|
||||
},
|
||||
}, nil)
|
||||
initErr := storage.Init()
|
||||
if initErr != nil {
|
||||
b.Fatal(initErr)
|
||||
}
|
||||
|
||||
const size = 1 << 20
|
||||
const chunkSize = 16 << 10
|
||||
var data = bytes.Repeat([]byte{'A'}, chunkSize)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
var writer = caches.NewMemoryWriter(storage, "a"+strconv.Itoa(rand.Int()), time.Now().Unix()+3600, 200, false, 0, 1<<30, func(valueItem *caches.MemoryItem) {
|
||||
})
|
||||
|
||||
for i := 0; i < size/chunkSize; i++ {
|
||||
_, err := writer.Write(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -5,12 +5,15 @@ package compressions_test
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/compressions"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkGzipWriter_Write(b *testing.B) {
|
||||
var data = []byte(strings.Repeat("A", 1024))
|
||||
var data = make([]byte, 1024)
|
||||
for i := 0; i < 1024; i++ {
|
||||
data[i] = 'A' + byte(i%26)
|
||||
}
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
var buf = &bytes.Buffer{}
|
||||
@@ -36,7 +39,12 @@ func BenchmarkGzipWriter_Write(b *testing.B) {
|
||||
}
|
||||
|
||||
func BenchmarkGzipWriter_Write_Parallel(b *testing.B) {
|
||||
var data = []byte(strings.Repeat("A", 1024))
|
||||
var data = make([]byte, 1024)
|
||||
for i := 0; i < 1024; i++ {
|
||||
data[i] = 'A' + byte(i%26)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.3.3"
|
||||
Version = "1.3.4"
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
@@ -16,4 +16,6 @@ const (
|
||||
|
||||
AccessLogSockName = "edge-node.accesslog"
|
||||
CacheGarbageSockName = "edge-node.cache.garbage"
|
||||
|
||||
EnableKVCacheStore = false // determine store cache keys in KVStore or sqlite
|
||||
)
|
||||
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var currentFirewall FirewallInterface
|
||||
@@ -35,6 +37,23 @@ func Firewall() FirewallInterface {
|
||||
return currentFirewall
|
||||
}
|
||||
|
||||
// http firewall
|
||||
{
|
||||
endpoint, _ := os.LookupEnv("EDGE_HTTP_FIREWALL_ENDPOINT")
|
||||
if len(endpoint) > 0 {
|
||||
var httpFirewall = NewHTTPFirewall(endpoint)
|
||||
for i := 0; i < 10; i++ {
|
||||
if httpFirewall.IsReady() {
|
||||
currentFirewall = httpFirewall
|
||||
remotelogs.Println("FIREWALL", "using http firewall '"+endpoint+"'")
|
||||
break
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
return httpFirewall
|
||||
}
|
||||
}
|
||||
|
||||
// nftables
|
||||
if runtime.GOOS == "linux" {
|
||||
nftables, err := NewNFTablesFirewall()
|
||||
|
||||
150
internal/firewalls/firewall_http.go
Normal file
150
internal/firewalls/firewall_http.go
Normal file
@@ -0,0 +1,150 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package firewalls
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type HTTPFirewall struct {
|
||||
client *http.Client
|
||||
endpoint string
|
||||
}
|
||||
|
||||
func NewHTTPFirewall(endpoint string) *HTTPFirewall {
|
||||
return &HTTPFirewall{
|
||||
client: http.DefaultClient,
|
||||
endpoint: endpoint,
|
||||
}
|
||||
}
|
||||
|
||||
// Name 名称
|
||||
func (this *HTTPFirewall) Name() string {
|
||||
result, err := this.get("/name", nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return result.GetString("name")
|
||||
}
|
||||
|
||||
// IsReady 是否已准备被调用
|
||||
func (this *HTTPFirewall) IsReady() bool {
|
||||
result, err := this.get("/isReady", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return result.GetBool("result")
|
||||
}
|
||||
|
||||
// IsMock 是否为模拟
|
||||
func (this *HTTPFirewall) IsMock() bool {
|
||||
result, err := this.get("/isMock", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return result.GetBool("result")
|
||||
}
|
||||
|
||||
// AllowPort 允许端口
|
||||
func (this *HTTPFirewall) AllowPort(port int, protocol string) error {
|
||||
_, err := this.get("/allowPort", map[string]string{
|
||||
"port": types.String(port),
|
||||
"protocol": protocol,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// RemovePort 删除端口
|
||||
func (this *HTTPFirewall) RemovePort(port int, protocol string) error {
|
||||
_, err := this.get("/removePort", map[string]string{
|
||||
"port": types.String(port),
|
||||
"protocol": protocol,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// RejectSourceIP 拒绝某个源IP连接
|
||||
func (this *HTTPFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
_, err := this.get("/rejectSourceIP", map[string]string{
|
||||
"ip": ip,
|
||||
"timeoutSeconds": types.String(timeoutSeconds),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// DropSourceIP 丢弃某个源IP数据
|
||||
// ip 要封禁的IP
|
||||
// timeoutSeconds 过期时间
|
||||
// async 是否异步
|
||||
func (this *HTTPFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
|
||||
var asyncString = "false"
|
||||
if async {
|
||||
asyncString = "true"
|
||||
}
|
||||
_, err := this.get("/dropSourceIP", map[string]string{
|
||||
"ip": ip,
|
||||
"timeoutSeconds": types.String(timeoutSeconds),
|
||||
"async": asyncString,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// RemoveSourceIP 删除某个源IP
|
||||
func (this *HTTPFirewall) RemoveSourceIP(ip string) error {
|
||||
_, err := this.get("/removeSourceIP", map[string]string{
|
||||
"ip": ip,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *HTTPFirewall) get(path string, args map[string]string) (result maps.Map, err error) {
|
||||
var urlString = this.endpoint + path
|
||||
if len(args) > 0 {
|
||||
var query = &url.Values{}
|
||||
for k, v := range args {
|
||||
query.Add(k, v)
|
||||
}
|
||||
urlString += "?" + query.Encode()
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodGet, urlString, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := this.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, errors.New("server response code '" + types.String(resp.StatusCode) + "'")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response failed: %w", err)
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
return maps.Map{}, nil
|
||||
}
|
||||
|
||||
result = maps.Map{}
|
||||
err = json.Unmarshal(data, &result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode response failed: %w", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -7,7 +7,6 @@ package nftables_test
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
|
||||
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -21,7 +20,10 @@ func TestConn_Test(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestConn_GetTable_NotFound(t *testing.T) {
|
||||
var conn = nftables.NewConn()
|
||||
conn, err := nftables.NewConn()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := conn.GetTable("a", nftables.TableFamilyIPv4)
|
||||
if err != nil {
|
||||
@@ -36,7 +38,10 @@ func TestConn_GetTable_NotFound(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestConn_GetTable(t *testing.T) {
|
||||
var conn = nftables.NewConn()
|
||||
conn, err := nftables.NewConn()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := conn.GetTable("myFilter", nftables.TableFamilyIPv4)
|
||||
if err != nil {
|
||||
@@ -51,7 +56,10 @@ func TestConn_GetTable(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestConn_AddTable(t *testing.T) {
|
||||
var conn = nftables.NewConn()
|
||||
conn, err := nftables.NewConn()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
{
|
||||
table, err := conn.AddIPv4Table("test_ipv4")
|
||||
@@ -70,8 +78,12 @@ func TestConn_AddTable(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestConn_DeleteTable(t *testing.T) {
|
||||
var conn = nftables.NewConn()
|
||||
err := conn.DeleteTable("test_ipv4", nftables.TableFamilyIPv4)
|
||||
conn, err := nftables.NewConn()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = conn.DeleteTable("test_ipv4", nftables.TableFamilyIPv4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -209,7 +209,7 @@ func (this *APIStream) handleWriteCache(message *pb.NodeStreamMessage) error {
|
||||
storage.AddToList(&caches.Item{
|
||||
Type: writer.ItemType(),
|
||||
Key: msg.Key,
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
HeaderSize: writer.HeaderSize(),
|
||||
BodySize: writer.BodySize(),
|
||||
})
|
||||
|
||||
@@ -67,7 +67,8 @@ func (this *HTTPAccessLogQueue) Push(accessLog *pb.HTTPAccessLog) {
|
||||
|
||||
// 上传访问日志
|
||||
func (this *HTTPAccessLogQueue) loop() error {
|
||||
var accessLogs = []*pb.HTTPAccessLog{}
|
||||
const maxLen = 2000
|
||||
var accessLogs = make([]*pb.HTTPAccessLog, 0, maxLen)
|
||||
var count = 0
|
||||
|
||||
Loop:
|
||||
@@ -78,7 +79,7 @@ Loop:
|
||||
count++
|
||||
|
||||
// 每次只提交 N 条访问日志,防止网络拥堵
|
||||
if count > 2000 {
|
||||
if count >= maxLen {
|
||||
break Loop
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -178,3 +178,55 @@ func BenchmarkHTTPAccessLogQueue_ToValidUTF8String(b *testing.B) {
|
||||
_ = strings.ToValidUTF8(s, "")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendAccessLogs(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
const count = 20000
|
||||
var a = make([]*pb.HTTPAccessLog, 0, count)
|
||||
for i := 0; i < b.N; i++ {
|
||||
a = append(a, &pb.HTTPAccessLog{
|
||||
RequestPath: "/hello/world",
|
||||
Host: "example.com",
|
||||
RequestBody: bytes.Repeat([]byte{'A'}, 1024),
|
||||
})
|
||||
if len(a) == count {
|
||||
a = make([]*pb.HTTPAccessLog, 0, count)
|
||||
}
|
||||
}
|
||||
|
||||
_ = len(a)
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
b.Log((stat2.TotalAlloc-stat1.TotalAlloc)>>20, "MB")
|
||||
}
|
||||
|
||||
func BenchmarkAppendAccessLogs2(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
const count = 20000
|
||||
var a = []*pb.HTTPAccessLog{}
|
||||
for i := 0; i < b.N; i++ {
|
||||
a = append(a, &pb.HTTPAccessLog{
|
||||
RequestPath: "/hello/world",
|
||||
Host: "example.com",
|
||||
RequestBody: bytes.Repeat([]byte{'A'}, 1024),
|
||||
})
|
||||
if len(a) == count {
|
||||
a = []*pb.HTTPAccessLog{}
|
||||
}
|
||||
}
|
||||
|
||||
_ = len(a)
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
b.Log((stat2.TotalAlloc-stat1.TotalAlloc)>>20, "MB")
|
||||
}
|
||||
|
||||
@@ -74,23 +74,41 @@ func (this *HTTPRequest) doMismatch() {
|
||||
}
|
||||
|
||||
// 处理当前连接
|
||||
if mismatchAction != nil && mismatchAction.Code == serverconfigs.DomainMismatchActionPage {
|
||||
if mismatchAction.Options != nil {
|
||||
this.writer.statusCode = statusCode
|
||||
var contentHTML = this.Format(mismatchAction.Options.GetString("contentHTML"))
|
||||
this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
|
||||
this.writer.WriteHeader(statusCode)
|
||||
_, _ = this.writer.Write([]byte(contentHTML))
|
||||
} else {
|
||||
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
|
||||
if mismatchAction != nil {
|
||||
if mismatchAction.Code == serverconfigs.DomainMismatchActionPage {
|
||||
if mismatchAction.Options != nil {
|
||||
this.writer.statusCode = statusCode
|
||||
var contentHTML = this.Format(mismatchAction.Options.GetString("contentHTML"))
|
||||
this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
|
||||
this.writer.WriteHeader(statusCode)
|
||||
_, _ = this.writer.Write([]byte(contentHTML))
|
||||
} else {
|
||||
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if mismatchAction.Code == serverconfigs.DomainMismatchActionRedirect {
|
||||
var url = this.Format(mismatchAction.Options.GetString("url"))
|
||||
if len(url) > 0 {
|
||||
httpRedirect(this.writer, this.RawReq, url, http.StatusTemporaryRedirect)
|
||||
} else {
|
||||
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if mismatchAction.Code == serverconfigs.DomainMismatchActionClose {
|
||||
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
|
||||
this.Close()
|
||||
return
|
||||
}
|
||||
return
|
||||
} else {
|
||||
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
|
||||
this.Close()
|
||||
return
|
||||
}
|
||||
|
||||
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
|
||||
this.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/minifiers"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
@@ -93,8 +94,8 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
if origin == nil {
|
||||
origin = this.reverseProxy.NextOrigin(requestCall)
|
||||
if origin != nil && origin.Id > 0 && (*failStatusCode >= 403 && *failStatusCode <= 404) && lists.ContainsInt64(failedOriginIds, origin.Id) {
|
||||
this.writeCode(*failStatusCode, "", "")
|
||||
return
|
||||
shouldRetry = false
|
||||
isLastRetry = true
|
||||
}
|
||||
}
|
||||
requestCall.CallResponseCallbacks(this.writer)
|
||||
@@ -442,7 +443,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
|
||||
// Page optimization
|
||||
if this.web.Optimization != nil && resp.Body != nil && this.cacheRef != nil /** must under cache **/ {
|
||||
err := this.web.Optimization.FilterResponse(this.URL(), resp)
|
||||
err := minifiers.MinifyResponse(this.web.Optimization, this.URL(), resp)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusBadGateway, "Page Optimization: fail to read content from origin", "内容优化:从源站读取内容失败", false)
|
||||
return
|
||||
|
||||
@@ -275,7 +275,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
}
|
||||
if err != nil {
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
|
||||
remotelogs.Warn("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -362,7 +362,7 @@ func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFi
|
||||
}
|
||||
if err != nil {
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
|
||||
remotelogs.Warn("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1168,7 +1168,7 @@ func (this *HTTPWriter) finishWebP() {
|
||||
this.cacheStorage.AddToList(&caches.Item{
|
||||
Type: webpCacheWriter.ItemType(),
|
||||
Key: webpCacheWriter.Key(),
|
||||
ExpiredAt: webpCacheWriter.ExpiredAt(),
|
||||
ExpiresAt: webpCacheWriter.ExpiredAt(),
|
||||
StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
|
||||
HeaderSize: webpCacheWriter.HeaderSize(),
|
||||
BodySize: webpCacheWriter.BodySize(),
|
||||
@@ -1207,7 +1207,7 @@ func (this *HTTPWriter) finishCache() {
|
||||
this.cacheStorage.AddToList(&caches.Item{
|
||||
Type: this.cacheWriter.ItemType(),
|
||||
Key: this.cacheWriter.Key(),
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
StaleAt: expiredAt + int64(this.calculateStaleLife()),
|
||||
HeaderSize: this.cacheWriter.HeaderSize(),
|
||||
BodySize: this.cacheWriter.BodySize(),
|
||||
@@ -1228,7 +1228,7 @@ func (this *HTTPWriter) finishCache() {
|
||||
this.cacheStorage.AddToList(&caches.Item{
|
||||
Type: this.cacheWriter.ItemType(),
|
||||
Key: this.cacheWriter.Key(),
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
StaleAt: expiredAt + int64(this.calculateStaleLife()),
|
||||
HeaderSize: this.cacheWriter.HeaderSize(),
|
||||
BodySize: this.cacheWriter.BodySize(),
|
||||
@@ -1251,7 +1251,7 @@ func (this *HTTPWriter) finishCompression() {
|
||||
this.cacheStorage.AddToList(&caches.Item{
|
||||
Type: this.compressionCacheWriter.ItemType(),
|
||||
Key: this.compressionCacheWriter.Key(),
|
||||
ExpiredAt: expiredAt,
|
||||
ExpiresAt: expiredAt,
|
||||
StaleAt: expiredAt + int64(this.calculateStaleLife()),
|
||||
HeaderSize: this.compressionCacheWriter.HeaderSize(),
|
||||
BodySize: this.compressionCacheWriter.BodySize(),
|
||||
|
||||
@@ -123,7 +123,7 @@ func (this *HTTPListener) ServeHTTPWithAddr(rawWriter http.ResponseWriter, rawRe
|
||||
|
||||
var globalServerConfig = sharedNodeConfig.GlobalServerConfig
|
||||
if globalServerConfig != nil && !globalServerConfig.HTTPAll.SupportsLowVersionHTTP && (rawReq.ProtoMajor < 1 /** 0.x **/ || (rawReq.ProtoMajor == 1 && rawReq.ProtoMinor == 0 /** 1.0 **/)) {
|
||||
http.Error(rawWriter, rawReq.Proto+" request is not supported.", http.StatusBadRequest)
|
||||
http.Error(rawWriter, rawReq.Proto+" request is not supported.", http.StatusHTTPVersionNotSupported)
|
||||
time.Sleep(1 * time.Second) // make connection slow down
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package utils
|
||||
package utils_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -10,7 +12,7 @@ func TestBytePool_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var pool = NewBytePool(32 * 1024)
|
||||
var pool = utils.NewBytePool(32 * 1024)
|
||||
for i := 0; i < 20480; i++ {
|
||||
pool.Put(make([]byte, 32*1024))
|
||||
}
|
||||
@@ -29,7 +31,7 @@ func TestBytePool_Memory(t *testing.T) {
|
||||
func BenchmarkBytePool_Get(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var pool = NewBytePool(1)
|
||||
var pool = utils.NewBytePool(1)
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -42,7 +44,7 @@ func BenchmarkBytePool_Get(b *testing.B) {
|
||||
func BenchmarkBytePool_Get_Parallel(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var pool = NewBytePool(1024)
|
||||
var pool = utils.NewBytePool(1024)
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
@@ -72,3 +74,20 @@ func BenchmarkBytePool_Get_Sync(b *testing.B) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkBytePool_Copy(b *testing.B) {
|
||||
var data = bytes.Repeat([]byte{'A'}, 8<<10)
|
||||
|
||||
var pool = &sync.Pool{
|
||||
New: func() any {
|
||||
return make([]byte, 8<<10)
|
||||
},
|
||||
}
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
var buf = pool.Get().([]byte)
|
||||
copy(buf, data)
|
||||
pool.Put(buf)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ type FastTime struct {
|
||||
unixTimeMilliString string
|
||||
ymd string
|
||||
round5Hi string
|
||||
hour int
|
||||
}
|
||||
|
||||
func NewFastTime() *FastTime {
|
||||
@@ -48,6 +49,7 @@ func NewFastTime() *FastTime {
|
||||
unixTimeMilliString: types.String(rawTime.UnixMilli()),
|
||||
ymd: timeutil.Format("Ymd", rawTime),
|
||||
round5Hi: timeutil.FormatTime("Hi", rawTime.Unix()/300*300),
|
||||
hour: rawTime.Hour(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,3 +93,7 @@ func (this *FastTime) Round5Hi() string {
|
||||
func (this *FastTime) Format(layout string) string {
|
||||
return timeutil.Format(layout, this.rawTime)
|
||||
}
|
||||
|
||||
func (this *FastTime) Hour() int {
|
||||
return this.hour
|
||||
}
|
||||
|
||||
@@ -38,6 +38,11 @@ func TestFastTime_Format(t *testing.T) {
|
||||
t.Log(now.Format("Y-m-d H:i:s"))
|
||||
}
|
||||
|
||||
func TestFastTime_Hour(t *testing.T) {
|
||||
var now = fasttime.Now()
|
||||
t.Log(now.Hour())
|
||||
}
|
||||
|
||||
func BenchmarkNewFastTime(b *testing.B) {
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
|
||||
68
internal/utils/kvstore/db.go
Normal file
68
internal/utils/kvstore/db.go
Normal file
@@ -0,0 +1,68 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
store *Store
|
||||
|
||||
name string
|
||||
namespace string
|
||||
tableMap map[string]TableInterface
|
||||
|
||||
locker sync.RWMutex
|
||||
}
|
||||
|
||||
func NewDB(store *Store, dbName string) (*DB, error) {
|
||||
if !IsValidName(dbName) {
|
||||
return nil, errors.New("invalid database name '" + dbName + "'")
|
||||
}
|
||||
|
||||
return &DB{
|
||||
store: store,
|
||||
name: dbName,
|
||||
namespace: "$" + dbName + "$",
|
||||
tableMap: map[string]TableInterface{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *DB) AddTable(table TableInterface) {
|
||||
table.SetNamespace([]byte(this.Namespace() + table.Name() + "$"))
|
||||
table.SetDB(this)
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
this.tableMap[table.Name()] = table
|
||||
}
|
||||
|
||||
func (this *DB) Name() string {
|
||||
return this.name
|
||||
}
|
||||
|
||||
func (this *DB) Namespace() string {
|
||||
return this.namespace
|
||||
}
|
||||
|
||||
func (this *DB) Store() *Store {
|
||||
return this.store
|
||||
}
|
||||
|
||||
func (this *DB) Close() error {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var lastErr error
|
||||
for _, table := range this.tableMap {
|
||||
err := table.Close()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
48
internal/utils/kvstore/db_test.go
Normal file
48
internal/utils/kvstore/db_test.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/cockroachdb/pebble"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewDB(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
_, err = store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testingStore = store
|
||||
testInspectDB(t)
|
||||
}
|
||||
|
||||
func testInspectDB(t *testing.T) {
|
||||
if testingStore == nil {
|
||||
return
|
||||
}
|
||||
it, err := testingStore.RawDB().NewIter(&pebble.IterOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = it.Close()
|
||||
}()
|
||||
|
||||
for it.First(); it.Valid(); it.Next() {
|
||||
valueBytes, valueErr := it.ValueAndErr()
|
||||
if valueErr != nil {
|
||||
t.Fatal(valueErr)
|
||||
}
|
||||
t.Log(string(it.Key()), "=>", string(valueBytes))
|
||||
}
|
||||
}
|
||||
18
internal/utils/kvstore/errors.go
Normal file
18
internal/utils/kvstore/errors.go
Normal file
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/cockroachdb/pebble"
|
||||
)
|
||||
|
||||
var ErrTableNotFound = errors.New("table not found")
|
||||
var ErrKeyTooLong = errors.New("too long key")
|
||||
|
||||
func IsKeyNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return errors.Is(err, pebble.ErrNotFound)
|
||||
}
|
||||
9
internal/utils/kvstore/item.go
Normal file
9
internal/utils/kvstore/item.go
Normal file
@@ -0,0 +1,9 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
type Item[T any] struct {
|
||||
Key string
|
||||
Value T
|
||||
FieldKey []byte
|
||||
}
|
||||
17
internal/utils/kvstore/iterator_options.go
Normal file
17
internal/utils/kvstore/iterator_options.go
Normal file
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import "github.com/cockroachdb/pebble"
|
||||
|
||||
type IteratorOptions struct {
|
||||
LowerBound []byte
|
||||
UpperBound []byte
|
||||
}
|
||||
|
||||
func (this *IteratorOptions) RawOptions() *pebble.IterOptions {
|
||||
return &pebble.IterOptions{
|
||||
LowerBound: this.LowerBound,
|
||||
UpperBound: this.UpperBound,
|
||||
}
|
||||
}
|
||||
17
internal/utils/kvstore/logger.go
Normal file
17
internal/utils/kvstore/logger.go
Normal file
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
type Logger struct {
|
||||
}
|
||||
|
||||
func NewLogger() *Logger {
|
||||
return &Logger{}
|
||||
}
|
||||
|
||||
func (this *Logger) Infof(format string, args ...any) {
|
||||
|
||||
}
|
||||
func (this *Logger) Fatalf(format string, args ...any) {
|
||||
|
||||
}
|
||||
9
internal/utils/kvstore/options.go
Normal file
9
internal/utils/kvstore/options.go
Normal file
@@ -0,0 +1,9 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import "github.com/cockroachdb/pebble"
|
||||
|
||||
var DefaultWriteOptions = &pebble.WriteOptions{
|
||||
Sync: false,
|
||||
}
|
||||
483
internal/utils/kvstore/query.go
Normal file
483
internal/utils/kvstore/query.go
Normal file
@@ -0,0 +1,483 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type DataType = int
|
||||
|
||||
const (
|
||||
DataTypeKey DataType = 1
|
||||
DataTypeField DataType = 2
|
||||
)
|
||||
|
||||
type QueryOperator int
|
||||
|
||||
const (
|
||||
QueryOperatorGt QueryOperator = 1
|
||||
QueryOperatorGte QueryOperator = 2
|
||||
QueryOperatorLt QueryOperator = 3
|
||||
QueryOperatorLte QueryOperator = 4
|
||||
)
|
||||
|
||||
type QueryOperatorInfo struct {
|
||||
Operator QueryOperator
|
||||
Value any
|
||||
}
|
||||
|
||||
type IteratorFunc[T any] func(tx *Tx[T], item Item[T]) (goNext bool, err error)
|
||||
|
||||
type Query[T any] struct {
|
||||
table *Table[T]
|
||||
tx *Tx[T]
|
||||
|
||||
dataType int
|
||||
offsetKey string
|
||||
limit int
|
||||
prefix string
|
||||
reverse bool
|
||||
forUpdate bool
|
||||
|
||||
keysOnly bool
|
||||
|
||||
fieldName string
|
||||
fieldReverse bool
|
||||
fieldOperators []QueryOperatorInfo
|
||||
fieldPrefix string
|
||||
fieldOffsetKey []byte
|
||||
}
|
||||
|
||||
func NewQuery[T any]() *Query[T] {
|
||||
return &Query[T]{
|
||||
limit: -1,
|
||||
dataType: DataTypeKey,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Query[T]) SetTable(table *Table[T]) *Query[T] {
|
||||
this.table = table
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) SetTx(tx *Tx[T]) *Query[T] {
|
||||
this.tx = tx
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) ForKey() *Query[T] {
|
||||
this.dataType = DataTypeKey
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) ForField() *Query[T] {
|
||||
this.dataType = DataTypeField
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) Limit(limit int) *Query[T] {
|
||||
this.limit = limit
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) Offset(offsetKey string) *Query[T] {
|
||||
this.offsetKey = offsetKey
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) Prefix(prefix string) *Query[T] {
|
||||
this.prefix = prefix
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) Desc() *Query[T] {
|
||||
this.reverse = true
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) ForUpdate() *Query[T] {
|
||||
this.forUpdate = true
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) KeysOnly() *Query[T] {
|
||||
this.keysOnly = true
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) FieldAsc(fieldName string) *Query[T] {
|
||||
this.fieldName = fieldName
|
||||
this.fieldReverse = false
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) FieldDesc(fieldName string) *Query[T] {
|
||||
this.fieldName = fieldName
|
||||
this.fieldReverse = true
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) FieldPrefix(fieldName string, fieldPrefix string) *Query[T] {
|
||||
this.fieldName = fieldName
|
||||
this.fieldPrefix = fieldPrefix
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *Query[T]) FieldOffset(fieldOffset []byte) *Query[T] {
|
||||
this.fieldOffsetKey = fieldOffset
|
||||
return this
|
||||
}
|
||||
|
||||
//func (this *Query[T]) FieldLt(value any) *Query[T] {
|
||||
// this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
|
||||
// Operator: QueryOperatorLt,
|
||||
// Value: value,
|
||||
// })
|
||||
// return this
|
||||
//}
|
||||
//
|
||||
//func (this *Query[T]) FieldLte(value any) *Query[T] {
|
||||
// this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
|
||||
// Operator: QueryOperatorLte,
|
||||
// Value: value,
|
||||
// })
|
||||
// return this
|
||||
//}
|
||||
//
|
||||
//func (this *Query[T]) FieldGt(value any) *Query[T] {
|
||||
// this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
|
||||
// Operator: QueryOperatorGt,
|
||||
// Value: value,
|
||||
// })
|
||||
// return this
|
||||
//}
|
||||
//
|
||||
//func (this *Query[T]) FieldGte(value any) *Query[T] {
|
||||
// this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
|
||||
// Operator: QueryOperatorGte,
|
||||
// Value: value,
|
||||
// })
|
||||
// return this
|
||||
//}
|
||||
|
||||
func (this *Query[T]) FindAll(fn IteratorFunc[T]) (err error) {
|
||||
defer func() {
|
||||
var panicErr = recover()
|
||||
if panicErr != nil {
|
||||
resultErr, ok := panicErr.(error)
|
||||
if ok {
|
||||
err = fmt.Errorf("execute query failed: %w", resultErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if this.tx != nil {
|
||||
defer func() {
|
||||
_ = this.tx.Close()
|
||||
}()
|
||||
|
||||
var itErr error
|
||||
if len(this.fieldName) == 0 {
|
||||
itErr = this.iterateKeys(fn)
|
||||
} else {
|
||||
itErr = this.iterateFields(fn)
|
||||
}
|
||||
if itErr != nil {
|
||||
return itErr
|
||||
}
|
||||
return this.tx.Commit()
|
||||
}
|
||||
|
||||
if this.table != nil {
|
||||
var txFn func(fn func(tx *Tx[T]) error) error
|
||||
if this.forUpdate {
|
||||
txFn = this.table.WriteTx
|
||||
} else {
|
||||
txFn = this.table.ReadTx
|
||||
}
|
||||
|
||||
return txFn(func(tx *Tx[T]) error {
|
||||
this.tx = tx
|
||||
|
||||
if len(this.fieldName) == 0 {
|
||||
return this.iterateKeys(fn)
|
||||
}
|
||||
return this.iterateFields(fn)
|
||||
})
|
||||
}
|
||||
|
||||
return errors.New("current query require 'table' or 'tx'")
|
||||
}
|
||||
|
||||
func (this *Query[T]) iterateKeys(fn IteratorFunc[T]) error {
|
||||
if this.limit == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var opt = &IteratorOptions{}
|
||||
|
||||
var prefix []byte
|
||||
switch this.dataType {
|
||||
case DataTypeKey:
|
||||
prefix = append(this.table.Namespace(), KeyPrefix...)
|
||||
case DataTypeField:
|
||||
prefix = append(this.table.Namespace(), FieldPrefix...)
|
||||
default:
|
||||
prefix = append(this.table.Namespace(), KeyPrefix...)
|
||||
}
|
||||
|
||||
var prefixLen = len(prefix)
|
||||
|
||||
if len(this.prefix) > 0 {
|
||||
prefix = append(prefix, this.prefix...)
|
||||
}
|
||||
|
||||
var offsetKey []byte
|
||||
if this.reverse {
|
||||
if len(this.offsetKey) > 0 {
|
||||
offsetKey = append(prefix, this.offsetKey...)
|
||||
} else {
|
||||
offsetKey = append(prefix, 0xFF)
|
||||
}
|
||||
|
||||
opt.LowerBound = prefix
|
||||
opt.UpperBound = offsetKey
|
||||
} else {
|
||||
if len(this.offsetKey) > 0 {
|
||||
offsetKey = append(prefix, this.offsetKey...)
|
||||
} else {
|
||||
offsetKey = prefix
|
||||
}
|
||||
opt.LowerBound = offsetKey
|
||||
opt.UpperBound = append(offsetKey, 0xFF)
|
||||
}
|
||||
|
||||
var hasOffsetKey = len(this.offsetKey) > 0
|
||||
|
||||
it, itErr := this.tx.NewIterator(opt)
|
||||
if itErr != nil {
|
||||
return itErr
|
||||
}
|
||||
defer func() {
|
||||
_ = it.Close()
|
||||
}()
|
||||
|
||||
var count int
|
||||
|
||||
var itemFn = func() (goNext bool, err error) {
|
||||
var keyBytes = it.Key()
|
||||
|
||||
// skip first offset key
|
||||
if hasOffsetKey {
|
||||
hasOffsetKey = false
|
||||
|
||||
if bytes.Equal(keyBytes, offsetKey) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// call fn
|
||||
var value T
|
||||
if !this.keysOnly {
|
||||
valueBytes, valueErr := it.ValueAndErr()
|
||||
if valueErr != nil {
|
||||
return false, valueErr
|
||||
}
|
||||
value, err = this.table.encoder.Decode(valueBytes)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
goNext, callbackErr := fn(this.tx, Item[T]{
|
||||
Key: string(keyBytes[prefixLen:]),
|
||||
Value: value,
|
||||
})
|
||||
if callbackErr != nil {
|
||||
return false, callbackErr
|
||||
}
|
||||
if !goNext {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// limit
|
||||
if this.limit > 0 {
|
||||
count++
|
||||
|
||||
if count >= this.limit {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if this.reverse {
|
||||
for it.Last(); it.Valid(); it.Prev() {
|
||||
goNext, itemErr := itemFn()
|
||||
if itemErr != nil {
|
||||
return itemErr
|
||||
}
|
||||
if !goNext {
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for it.First(); it.Valid(); it.Next() {
|
||||
goNext, itemErr := itemFn()
|
||||
if itemErr != nil {
|
||||
return itemErr
|
||||
}
|
||||
if !goNext {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Query[T]) iterateFields(fn IteratorFunc[T]) error {
|
||||
if this.limit == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var hasOffsetKey = len(this.offsetKey) > 0 || len(this.fieldOffsetKey) > 0
|
||||
|
||||
var opt = &IteratorOptions{}
|
||||
|
||||
var prefix = this.table.FieldKey(this.fieldName)
|
||||
prefix = append(prefix, '$')
|
||||
|
||||
if len(this.fieldPrefix) > 0 {
|
||||
prefix = append(prefix, this.fieldPrefix...)
|
||||
}
|
||||
|
||||
var offsetKey []byte
|
||||
if this.fieldReverse {
|
||||
if len(this.fieldOffsetKey) > 0 {
|
||||
offsetKey = this.fieldOffsetKey
|
||||
} else if len(this.offsetKey) > 0 {
|
||||
offsetKey = append(prefix, this.offsetKey...)
|
||||
} else {
|
||||
offsetKey = append(prefix, 0xFF)
|
||||
}
|
||||
opt.LowerBound = prefix
|
||||
opt.UpperBound = offsetKey
|
||||
} else {
|
||||
if len(this.fieldOffsetKey) > 0 {
|
||||
offsetKey = this.fieldOffsetKey
|
||||
} else if len(this.offsetKey) > 0 {
|
||||
offsetKey = append(prefix, this.offsetKey...)
|
||||
offsetKey = append(offsetKey, 0)
|
||||
} else {
|
||||
offsetKey = prefix
|
||||
}
|
||||
|
||||
opt.LowerBound = offsetKey
|
||||
opt.UpperBound = append(prefix, 0xFF)
|
||||
}
|
||||
|
||||
it, itErr := this.tx.NewIterator(opt)
|
||||
if itErr != nil {
|
||||
return itErr
|
||||
}
|
||||
defer func() {
|
||||
_ = it.Close()
|
||||
}()
|
||||
|
||||
var count int
|
||||
|
||||
var itemFn = func() (goNext bool, err error) {
|
||||
var fieldKeyBytes = it.Key()
|
||||
|
||||
fieldValueBytes, keyBytes, decodeKeyErr := this.table.DecodeFieldKey(this.fieldName, fieldKeyBytes)
|
||||
if decodeKeyErr != nil {
|
||||
return false, decodeKeyErr
|
||||
}
|
||||
|
||||
// skip first offset key
|
||||
if hasOffsetKey {
|
||||
hasOffsetKey = false
|
||||
|
||||
if (len(this.fieldOffsetKey) > 0 && bytes.Equal(fieldKeyBytes, this.fieldOffsetKey)) ||
|
||||
bytes.Equal(fieldValueBytes, []byte(this.offsetKey)) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 执行operators
|
||||
if len(this.fieldOperators) > 0 {
|
||||
if !this.matchOperators(fieldValueBytes) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
var resultItem = Item[T]{
|
||||
Key: string(keyBytes),
|
||||
FieldKey: fieldKeyBytes,
|
||||
}
|
||||
if !this.keysOnly {
|
||||
value, getErr := this.table.getWithKeyBytes(this.tx, this.table.FullKeyBytes(keyBytes))
|
||||
if getErr != nil {
|
||||
if IsKeyNotFound(getErr) {
|
||||
return true, nil
|
||||
}
|
||||
return false, getErr
|
||||
}
|
||||
|
||||
resultItem.Value = value
|
||||
}
|
||||
|
||||
goNext, err = fn(this.tx, resultItem)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !goNext {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// limit
|
||||
if this.limit > 0 {
|
||||
count++
|
||||
|
||||
if count >= this.limit {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if this.reverse {
|
||||
for it.Last(); it.Valid(); it.Prev() {
|
||||
goNext, itemErr := itemFn()
|
||||
if itemErr != nil {
|
||||
return itemErr
|
||||
}
|
||||
if !goNext {
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for it.First(); it.Valid(); it.Next() {
|
||||
goNext, itemErr := itemFn()
|
||||
if itemErr != nil {
|
||||
return itemErr
|
||||
}
|
||||
if !goNext {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Query[T]) matchOperators(fieldValueBytes []byte) bool {
|
||||
// TODO
|
||||
return true
|
||||
}
|
||||
276
internal/utils/kvstore/query_test.go
Normal file
276
internal/utils/kvstore/query_test.go
Normal file
@@ -0,0 +1,276 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQuery_FindAll(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
err := table.
|
||||
Query().
|
||||
Limit(10).
|
||||
//Offset("a1000").
|
||||
//Desc().
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log("key:", item.Key, "value:", item.Value)
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuery_FindAll_Break(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
var count int
|
||||
err := table.
|
||||
Query().
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log("key:", item.Key, "value:", item.Value)
|
||||
count++
|
||||
|
||||
if count > 2 {
|
||||
// break test
|
||||
_ = table.DB().Store().Close()
|
||||
}
|
||||
|
||||
return count < 3, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuery_FindAll_Break_Closed(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
var count int
|
||||
err := table.
|
||||
Query().
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log("key:", item.Key, "value:", item.Value)
|
||||
count++
|
||||
|
||||
if count > 2 {
|
||||
// break test
|
||||
_ = table.DB().Store().Close()
|
||||
}
|
||||
|
||||
return count < 3, nil
|
||||
})
|
||||
t.Log("expected error:", err)
|
||||
a.IsTrue(err != nil)
|
||||
}
|
||||
|
||||
func TestQuery_FindAll_Desc(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
err := table.Query().
|
||||
Desc().
|
||||
Limit(10).
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log("key:", item.Key, "value:", item.Value)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuery_FindAll_Offset(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
{
|
||||
t.Log("=== forward ===")
|
||||
err := table.Query().
|
||||
Offset("a3").
|
||||
Limit(10).
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log("key:", item.Key, "value:", item.Value)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
t.Log("=== backward ===")
|
||||
err := table.Query().
|
||||
Desc().
|
||||
Offset("a3").
|
||||
Limit(10).
|
||||
//KeyOnly().
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log("key:", item.Key, "value:", item.Value)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuery_FindAll_Count(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var count int
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms", "qps:", fmt.Sprintf("%.2fM/s", float64(count)/costSeconds/1_000_000))
|
||||
}()
|
||||
|
||||
err := table.
|
||||
Query().
|
||||
KeysOnly().
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
count++
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("count:", count)
|
||||
}
|
||||
|
||||
func TestQuery_FindAll_Field(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms", "qps:", int(1/costSeconds))
|
||||
}()
|
||||
|
||||
var lastFieldKey []byte
|
||||
|
||||
t.Log("=======")
|
||||
{
|
||||
err := table.
|
||||
Query().
|
||||
FieldAsc("expiresAt").
|
||||
//KeysOnly().
|
||||
//FieldLt(1710848959).
|
||||
Limit(3).
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log(item.Key, "=>", item.Value)
|
||||
lastFieldKey = item.FieldKey
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
t.Log("=======")
|
||||
{
|
||||
err := table.
|
||||
Query().
|
||||
FieldAsc("expiresAt").
|
||||
//KeysOnly().
|
||||
//FieldLt(1710848959).
|
||||
FieldOffset(lastFieldKey).
|
||||
Limit(3).
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log(item.Key, "=>", item.Value)
|
||||
lastFieldKey = item.FieldKey
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuery_FindAll_Field_Many(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms", "qps:", int(1/costSeconds))
|
||||
}()
|
||||
|
||||
err := table.
|
||||
Query().
|
||||
FieldAsc("expiresAt").
|
||||
KeysOnly().
|
||||
Limit(1000).
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
t.Log(item.Key, "=>", item.Value)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkQuery_FindAll(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("db1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[*testCachedItem]("cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
err = table.Query().
|
||||
//Limit(100).
|
||||
FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
169
internal/utils/kvstore/store.go
Normal file
169
internal/utils/kvstore/store.go
Normal file
@@ -0,0 +1,169 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/cockroachdb/pebble"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const StoreSuffix = ".store"
|
||||
|
||||
type Store struct {
|
||||
name string
|
||||
|
||||
path string
|
||||
rawDB *pebble.DB
|
||||
|
||||
isClosed bool
|
||||
|
||||
dbs []*DB
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewStore create store with name
|
||||
func NewStore(storeName string) (*Store, error) {
|
||||
if !IsValidName(storeName) {
|
||||
return nil, errors.New("invalid store name '" + storeName + "'")
|
||||
}
|
||||
|
||||
var root = Tea.Root + "/data/stores"
|
||||
_, err := os.Stat(root)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(root, 0777)
|
||||
}
|
||||
|
||||
return &Store{
|
||||
name: storeName,
|
||||
path: Tea.Root + "/data/stores/" + storeName + StoreSuffix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func OpenStore(storeName string) (*Store, error) {
|
||||
store, err := NewStore(storeName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = store.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func OpenStoreDir(dir string, storeName string) (*Store, error) {
|
||||
if !IsValidName(storeName) {
|
||||
return nil, errors.New("invalid store name '" + storeName + "'")
|
||||
}
|
||||
|
||||
_, err := os.Stat(dir)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dir, 0777)
|
||||
}
|
||||
|
||||
dir = strings.TrimSuffix(dir, "/")
|
||||
|
||||
var store = &Store{
|
||||
name: storeName,
|
||||
path: dir + "/" + storeName + StoreSuffix,
|
||||
}
|
||||
|
||||
err = store.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (this *Store) Open() error {
|
||||
var opt = &pebble.Options{
|
||||
Logger: NewLogger(),
|
||||
}
|
||||
|
||||
// TODO 需要修改 BytesPerSync 和 WALBytesPerSync 等等默认参数
|
||||
|
||||
rawDB, err := pebble.Open(this.path, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.rawDB = rawDB
|
||||
|
||||
// events
|
||||
events.OnKey(events.EventQuit, fmt.Sprintf("kvstore_%p", this), func() {
|
||||
_ = this.Close()
|
||||
})
|
||||
events.OnKey(events.EventTerminated, fmt.Sprintf("kvstore_%p", this), func() {
|
||||
_ = this.Close()
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Store) Set(keyBytes []byte, valueBytes []byte) error {
|
||||
return this.rawDB.Set(keyBytes, valueBytes, DefaultWriteOptions)
|
||||
}
|
||||
|
||||
func (this *Store) Get(keyBytes []byte) (valueBytes []byte, closer io.Closer, err error) {
|
||||
return this.rawDB.Get(keyBytes)
|
||||
}
|
||||
|
||||
func (this *Store) Delete(keyBytes []byte) error {
|
||||
return this.rawDB.Delete(keyBytes, DefaultWriteOptions)
|
||||
}
|
||||
|
||||
func (this *Store) NewDB(dbName string) (*DB, error) {
|
||||
db, err := NewDB(this, dbName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
|
||||
this.dbs = append(this.dbs, db)
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (this *Store) RawDB() *pebble.DB {
|
||||
return this.rawDB
|
||||
}
|
||||
|
||||
func (this *Store) Close() error {
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
this.mu.Lock()
|
||||
var lastErr error
|
||||
for _, db := range this.dbs {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
|
||||
this.mu.Unlock()
|
||||
|
||||
if this.rawDB != nil {
|
||||
this.isClosed = true
|
||||
err := this.rawDB.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (this *Store) IsClosed() bool {
|
||||
return this.isClosed
|
||||
}
|
||||
162
internal/utils/kvstore/store_test.go
Normal file
162
internal/utils/kvstore/store_test.go
Normal file
@@ -0,0 +1,162 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/cockroachdb/pebble"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
m.Run()
|
||||
|
||||
if testingStore != nil {
|
||||
_ = testingStore.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_Open(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
t.Log("opened")
|
||||
}
|
||||
|
||||
func TestStore_RawDB(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
err = store.RawDB().Set([]byte("hello"), []byte("world"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenStoreDir(t *testing.T) {
|
||||
store, err := kvstore.OpenStoreDir(Tea.Root+"/data/stores", "test3")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
t.Log("opened")
|
||||
|
||||
_ = store
|
||||
}
|
||||
|
||||
func TestStore_CloseTwice(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
for i := 0; i < 3; i++ {
|
||||
err = store.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func TestStore_Count(t *testing.T) {
|
||||
testCountStore(t)
|
||||
}
|
||||
|
||||
var testingStore *kvstore.Store
|
||||
|
||||
func testOpenStore(t *testing.T) *kvstore.DB {
|
||||
var err error
|
||||
testingStore, err = kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db, err := testingStore.NewDB("db1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
|
||||
func testOpenStoreTable[T any](t *testing.T, tableName string, encoder kvstore.ValueEncoder[T]) *kvstore.Table[T] {
|
||||
var err error
|
||||
|
||||
var before = time.Now()
|
||||
testingStore, err = kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("store open cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
|
||||
db, err := testingStore.NewDB("db1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[T](tableName, encoder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.AddTable(table)
|
||||
|
||||
return table
|
||||
}
|
||||
|
||||
func testOpenStoreTableForBenchmark[T any](t *testing.B, tableName string, encoder kvstore.ValueEncoder[T]) *kvstore.Table[T] {
|
||||
var err error
|
||||
testingStore, err = kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db, err := testingStore.NewDB("db1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[T](tableName, encoder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.AddTable(table)
|
||||
|
||||
return table
|
||||
}
|
||||
|
||||
func testCountStore(t *testing.T) {
|
||||
var err error
|
||||
testingStore, err = kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
it, err := testingStore.RawDB().NewIter(&pebble.IterOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = it.Close()
|
||||
}()
|
||||
for it.First(); it.Valid(); it.Next() {
|
||||
count++
|
||||
}
|
||||
t.Log("count:", count)
|
||||
}
|
||||
383
internal/utils/kvstore/table.go
Normal file
383
internal/utils/kvstore/table.go
Normal file
@@ -0,0 +1,383 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"github.com/cockroachdb/pebble"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
KeyPrefix = "K$"
|
||||
KeyMaxLength = 8 << 10
|
||||
|
||||
FieldPrefix = "F$"
|
||||
|
||||
MaxBatchKeys = 8 << 10 // TODO not implemented
|
||||
)
|
||||
|
||||
type Table[T any] struct {
|
||||
name string
|
||||
rawNamespace []byte
|
||||
db *DB
|
||||
encoder ValueEncoder[T]
|
||||
fieldNames []string
|
||||
|
||||
mu *sync.RWMutex
|
||||
}
|
||||
|
||||
func NewTable[T any](tableName string, encoder ValueEncoder[T]) (*Table[T], error) {
|
||||
if !IsValidName(tableName) {
|
||||
return nil, errors.New("invalid table name '" + tableName + "'")
|
||||
}
|
||||
|
||||
return &Table[T]{
|
||||
name: tableName,
|
||||
encoder: encoder,
|
||||
mu: &sync.RWMutex{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *Table[T]) Name() string {
|
||||
return this.name
|
||||
}
|
||||
|
||||
func (this *Table[T]) Namespace() []byte {
|
||||
var dest = make([]byte, len(this.rawNamespace))
|
||||
copy(dest, this.rawNamespace)
|
||||
return dest
|
||||
}
|
||||
|
||||
func (this *Table[T]) SetNamespace(namespace []byte) {
|
||||
this.rawNamespace = namespace
|
||||
}
|
||||
|
||||
func (this *Table[T]) SetDB(db *DB) {
|
||||
this.db = db
|
||||
}
|
||||
|
||||
func (this *Table[T]) DB() *DB {
|
||||
return this.db
|
||||
}
|
||||
|
||||
func (this *Table[T]) Set(key string, value T) error {
|
||||
if len(key) > KeyMaxLength {
|
||||
return ErrKeyTooLong
|
||||
}
|
||||
|
||||
valueBytes, err := this.encoder.Encode(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return this.WriteTx(func(tx *Tx[T]) error {
|
||||
return this.set(tx, key, valueBytes, value)
|
||||
})
|
||||
}
|
||||
|
||||
// ComposeFieldKey compose field key
|
||||
// $Namespace$FieldName$FieldValueSeparatorKeyValueFieldLength[2]
|
||||
func (this *Table[T]) ComposeFieldKey(keyBytes []byte, fieldName string, fieldValueBytes []byte) []byte {
|
||||
// TODO use 'make()' and 'copy()' to pre-alloc memory space
|
||||
var b = make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(b, uint16(len(fieldValueBytes)))
|
||||
var fieldKey = append(this.FieldKey(fieldName), '$') // namespace
|
||||
fieldKey = append(fieldKey, fieldValueBytes...) // field value
|
||||
fieldKey = append(fieldKey, 0, 0) // separator
|
||||
fieldKey = append(fieldKey, keyBytes...) // key value
|
||||
fieldKey = append(fieldKey, b...) // field value length
|
||||
return fieldKey
|
||||
}
|
||||
|
||||
func (this *Table[T]) Exist(key string) (found bool, err error) {
|
||||
_, closer, err := this.db.store.rawDB.Get(this.FullKey(key))
|
||||
if err != nil {
|
||||
if IsKeyNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
defer func() {
|
||||
_ = closer.Close()
|
||||
}()
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (this *Table[T]) Get(key string) (value T, err error) {
|
||||
err = this.ReadTx(func(tx *Tx[T]) error {
|
||||
resultValue, getErr := this.get(tx, key)
|
||||
if getErr == nil {
|
||||
value = resultValue
|
||||
}
|
||||
return getErr
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Table[T]) Delete(key ...string) error {
|
||||
if len(key) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return this.WriteTx(func(tx *Tx[T]) error {
|
||||
return this.deleteKeys(tx, key...)
|
||||
})
|
||||
}
|
||||
|
||||
func (this *Table[T]) ReadTx(fn func(tx *Tx[T]) error) error {
|
||||
var tx = NewTx[T](this, true)
|
||||
defer func() {
|
||||
_ = tx.Close()
|
||||
}()
|
||||
|
||||
err := fn(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (this *Table[T]) WriteTx(fn func(tx *Tx[T]) error) error {
|
||||
var tx = NewTx[T](this, false)
|
||||
defer func() {
|
||||
_ = tx.Close()
|
||||
}()
|
||||
|
||||
err := fn(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (this *Table[T]) Truncate() error {
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
|
||||
return this.db.store.rawDB.DeleteRange(this.Namespace(), append(this.Namespace(), 0xFF), DefaultWriteOptions)
|
||||
}
|
||||
|
||||
func (this *Table[T]) Query() *Query[T] {
|
||||
var query = NewQuery[T]()
|
||||
query.SetTable(this)
|
||||
return query
|
||||
}
|
||||
|
||||
func (this *Table[T]) Count() (int64, error) {
|
||||
var count int64
|
||||
|
||||
var begin = this.FullKeyBytes(nil)
|
||||
it, err := this.db.store.rawDB.NewIter(&pebble.IterOptions{
|
||||
LowerBound: begin,
|
||||
UpperBound: append(begin, 0xFF),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
_ = it.Close()
|
||||
}()
|
||||
|
||||
for it.First(); it.Valid(); it.Next() {
|
||||
count++
|
||||
}
|
||||
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (this *Table[T]) FullKey(realKey string) []byte {
|
||||
return append(this.Namespace(), KeyPrefix+realKey...)
|
||||
}
|
||||
|
||||
func (this *Table[T]) FullKeyBytes(realKeyBytes []byte) []byte {
|
||||
var k = append(this.Namespace(), KeyPrefix...)
|
||||
k = append(k, realKeyBytes...)
|
||||
return k
|
||||
}
|
||||
|
||||
func (this *Table[T]) FieldKey(fieldName string) []byte {
|
||||
var data = append(this.Namespace(), FieldPrefix...)
|
||||
data = append(data, fieldName...)
|
||||
return data
|
||||
}
|
||||
|
||||
func (this *Table[T]) DecodeFieldKey(fieldName string, fieldKey []byte) (fieldValue []byte, key []byte, err error) {
|
||||
var l = len(fieldKey)
|
||||
var baseLen = len(this.FieldKey(fieldName)) + 1 /** $ **/ + 2 /** separator length **/ + 2 /** field length **/
|
||||
if l < baseLen {
|
||||
err = errors.New("invalid field key")
|
||||
return
|
||||
}
|
||||
|
||||
var fieldValueLen = binary.BigEndian.Uint16(fieldKey[l-2:])
|
||||
var data = fieldKey[baseLen-4 : l-2]
|
||||
|
||||
fieldValue = data[:fieldValueLen]
|
||||
key = data[fieldValueLen+2: /** separator length **/]
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Table[T]) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Table[T]) deleteKeys(tx *Tx[T], key ...string) error {
|
||||
var batch = tx.batch
|
||||
|
||||
for _, singleKey := range key {
|
||||
var keyErr = func(singleKey string) error {
|
||||
var keyBytes = this.FullKey(singleKey)
|
||||
|
||||
// delete field values
|
||||
if len(this.fieldNames) > 0 {
|
||||
valueBytes, closer, getErr := batch.Get(keyBytes)
|
||||
if getErr != nil {
|
||||
if IsKeyNotFound(getErr) {
|
||||
return nil
|
||||
}
|
||||
return getErr
|
||||
}
|
||||
defer func() {
|
||||
_ = closer.Close()
|
||||
}()
|
||||
|
||||
value, decodeErr := this.encoder.Decode(valueBytes)
|
||||
if decodeErr != nil {
|
||||
return decodeErr
|
||||
}
|
||||
|
||||
for _, fieldName := range this.fieldNames {
|
||||
fieldValueBytes, fieldErr := this.encoder.EncodeField(value, fieldName)
|
||||
if fieldErr != nil {
|
||||
return fieldErr
|
||||
}
|
||||
|
||||
deleteKeyErr := batch.Delete(this.ComposeFieldKey([]byte(singleKey), fieldName, fieldValueBytes), DefaultWriteOptions)
|
||||
if deleteKeyErr != nil {
|
||||
return deleteKeyErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err := batch.Delete(keyBytes, DefaultWriteOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}(singleKey)
|
||||
if keyErr != nil {
|
||||
return keyErr
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Table[T]) set(tx *Tx[T], key string, valueBytes []byte, value T) error {
|
||||
var keyBytes = this.FullKey(key)
|
||||
|
||||
var batch = tx.batch
|
||||
|
||||
// read old value
|
||||
var oldValue T
|
||||
var oldFound bool
|
||||
var countFields = len(this.fieldNames)
|
||||
if countFields > 0 {
|
||||
oldValueBytes, closer, getErr := batch.Get(keyBytes)
|
||||
if getErr != nil {
|
||||
if !IsKeyNotFound(getErr) {
|
||||
return getErr
|
||||
}
|
||||
} else {
|
||||
defer func() {
|
||||
_ = closer.Close()
|
||||
}()
|
||||
|
||||
var decodeErr error
|
||||
oldValue, decodeErr = this.encoder.Decode(oldValueBytes)
|
||||
if decodeErr != nil {
|
||||
return decodeErr
|
||||
}
|
||||
oldFound = true
|
||||
}
|
||||
}
|
||||
|
||||
setErr := batch.Set(keyBytes, valueBytes, DefaultWriteOptions)
|
||||
if setErr != nil {
|
||||
return setErr
|
||||
}
|
||||
|
||||
// process fields
|
||||
if countFields > 0 {
|
||||
// add new field keys
|
||||
for _, fieldName := range this.fieldNames {
|
||||
// 把EncodeField放在TX里,是为了节约内存
|
||||
fieldValueBytes, fieldErr := this.encoder.EncodeField(value, fieldName)
|
||||
if fieldErr != nil {
|
||||
return fieldErr
|
||||
}
|
||||
|
||||
if len(fieldValueBytes) > 8<<10 {
|
||||
return errors.New("field value too long: " + types.String(len(fieldValueBytes)))
|
||||
}
|
||||
|
||||
var newFieldKeyBytes = this.ComposeFieldKey([]byte(key), fieldName, fieldValueBytes)
|
||||
|
||||
// delete old field key
|
||||
if oldFound {
|
||||
oldFieldValueBytes, oldFieldErr := this.encoder.EncodeField(oldValue, fieldName)
|
||||
if oldFieldErr != nil {
|
||||
return oldFieldErr
|
||||
}
|
||||
var oldFieldKeyBytes = this.ComposeFieldKey([]byte(key), fieldName, oldFieldValueBytes)
|
||||
if bytes.Equal(oldFieldKeyBytes, newFieldKeyBytes) {
|
||||
// skip the field
|
||||
continue
|
||||
}
|
||||
deleteFieldErr := batch.Delete(oldFieldKeyBytes, DefaultWriteOptions)
|
||||
if deleteFieldErr != nil {
|
||||
return deleteFieldErr
|
||||
}
|
||||
}
|
||||
|
||||
// set new field key
|
||||
setFieldErr := batch.Set(newFieldKeyBytes, nil, DefaultWriteOptions)
|
||||
if setFieldErr != nil {
|
||||
return setFieldErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Table[T]) get(tx *Tx[T], key string) (value T, err error) {
|
||||
return this.getWithKeyBytes(tx, this.FullKey(key))
|
||||
}
|
||||
|
||||
func (this *Table[T]) getWithKeyBytes(tx *Tx[T], keyBytes []byte) (value T, err error) {
|
||||
valueBytes, closer, err := tx.batch.Get(keyBytes)
|
||||
if err != nil {
|
||||
return value, err
|
||||
}
|
||||
defer func() {
|
||||
_ = closer.Close()
|
||||
}()
|
||||
|
||||
resultValue, decodeErr := this.encoder.Decode(valueBytes)
|
||||
if decodeErr != nil {
|
||||
return value, decodeErr
|
||||
}
|
||||
value = resultValue
|
||||
return
|
||||
}
|
||||
33
internal/utils/kvstore/table_counter.go
Normal file
33
internal/utils/kvstore/table_counter.go
Normal file
@@ -0,0 +1,33 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
type CounterTable[T int64 | uint64] struct {
|
||||
*Table[T]
|
||||
}
|
||||
|
||||
func NewCounterTable[T int64 | uint64](name string) (*CounterTable[T], error) {
|
||||
table, err := NewTable[T](name, NewIntValueEncoder[T]())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &CounterTable[T]{
|
||||
Table: table,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *CounterTable[T]) Increase(key string, delta T) (newValue T, err error) {
|
||||
err = this.Table.WriteTx(func(tx *Tx[T]) error {
|
||||
value, getErr := tx.Get(key)
|
||||
if getErr != nil {
|
||||
if !IsKeyNotFound(getErr) {
|
||||
return getErr
|
||||
}
|
||||
}
|
||||
|
||||
newValue = value + delta
|
||||
return tx.Set(key, newValue)
|
||||
})
|
||||
return
|
||||
}
|
||||
80
internal/utils/kvstore/table_counter_test.go
Normal file
80
internal/utils/kvstore/table_counter_test.go
Normal file
@@ -0,0 +1,80 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCounterTable_Increase(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewCounterTable[uint64]("users_counter")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
count, err := table.Increase("counter", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(count)
|
||||
}
|
||||
|
||||
func BenchmarkCounterTable_Increase(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewCounterTable[uint64]("users_counter")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
defer func() {
|
||||
count, incrErr := table.Increase("counter", 1)
|
||||
if incrErr != nil {
|
||||
b.Fatal(incrErr)
|
||||
}
|
||||
b.Log(count)
|
||||
}()
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_, incrErr := table.Increase("counter", 1)
|
||||
if incrErr != nil {
|
||||
b.Fatal(incrErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
41
internal/utils/kvstore/table_field.go
Normal file
41
internal/utils/kvstore/table_field.go
Normal file
@@ -0,0 +1,41 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
func (this *Table[T]) AddField(fieldName string) error {
|
||||
if !IsValidName(fieldName) {
|
||||
return errors.New("invalid field name '" + fieldName + "'")
|
||||
}
|
||||
|
||||
// check existence
|
||||
for _, field := range this.fieldNames {
|
||||
if field == fieldName {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
this.fieldNames = append(this.fieldNames, fieldName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Table[T]) AddFields(fieldName ...string) error {
|
||||
for _, subFieldName := range fieldName {
|
||||
err := this.AddField(subFieldName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Table[T]) DropField(fieldName string) error {
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
|
||||
var start = this.FieldKey(fieldName + "$")
|
||||
return this.db.store.rawDB.DeleteRange(start, append(start, 0xFF), DefaultWriteOptions)
|
||||
}
|
||||
251
internal/utils/kvstore/table_field_test.go
Normal file
251
internal/utils/kvstore/table_field_test.go
Normal file
@@ -0,0 +1,251 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type testCachedItem struct {
|
||||
Hash string `json:"1"` // as key
|
||||
URL string `json:"2"`
|
||||
ExpiresAt int64 `json:"3"`
|
||||
Tag string `json:"tag"`
|
||||
HeaderSize int64 `json:"headerSize"`
|
||||
BodySize int64 `json:"bodySize"`
|
||||
MetaSize int `json:"metaSize"`
|
||||
StaleAt int64 `json:"staleAt"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
Host string `json:"host"`
|
||||
ServerId int64 `json:"serverId"`
|
||||
}
|
||||
|
||||
type testCacheItemEncoder[T interface{ *testCachedItem }] struct {
|
||||
kvstore.BaseObjectEncoder[T]
|
||||
}
|
||||
|
||||
func (this *testCacheItemEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
switch fieldName {
|
||||
case "expiresAt":
|
||||
var b = make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(b, uint32(any(value).(*testCachedItem).ExpiresAt))
|
||||
return b, nil
|
||||
case "staleAt":
|
||||
var b = make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(b, uint32(any(value).(*testCachedItem).StaleAt))
|
||||
return b, nil
|
||||
case "url":
|
||||
return []byte(any(value).(*testCachedItem).URL), nil
|
||||
}
|
||||
return nil, errors.New("EncodeField: invalid field name '" + fieldName + "'")
|
||||
}
|
||||
|
||||
func TestTable_AddField(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
err := table.AddFields("expiresAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var before = time.Now()
|
||||
for _, item := range []*testCachedItem{
|
||||
{
|
||||
Hash: "a1",
|
||||
URL: "https://example.com/a1",
|
||||
ExpiresAt: 1710832067,
|
||||
},
|
||||
{
|
||||
Hash: "a5",
|
||||
URL: "https://example.com/a5",
|
||||
ExpiresAt: time.Now().Unix() + 7200,
|
||||
},
|
||||
{
|
||||
Hash: "a4",
|
||||
URL: "https://example.com/a4",
|
||||
ExpiresAt: time.Now().Unix() + 86400,
|
||||
},
|
||||
{
|
||||
Hash: "a3",
|
||||
URL: "https://example.com/a3",
|
||||
ExpiresAt: time.Now().Unix() + 1800,
|
||||
},
|
||||
{
|
||||
Hash: "a2",
|
||||
URL: "https://example.com/a2",
|
||||
ExpiresAt: time.Now().Unix() + 365*86400,
|
||||
},
|
||||
} {
|
||||
err = table.Set(item.Hash, item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
t.Log("set cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
|
||||
testInspectDB(t)
|
||||
}
|
||||
|
||||
func TestTable_AddField_Many(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
//runtime.GOMAXPROCS(1)
|
||||
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
{
|
||||
err := table.AddFields("expiresAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
err := table.AddFields("staleAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
err := table.AddFields("url")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var before = time.Now()
|
||||
const from = 0
|
||||
const count = 4_000_000
|
||||
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms", "qps:", int64(float64(count)/costSeconds))
|
||||
}()
|
||||
|
||||
for i := from; i < from+count; i++ {
|
||||
var item = &testCachedItem{
|
||||
Hash: "a" + strconv.Itoa(i),
|
||||
URL: "https://example.com/a" + strconv.Itoa(i),
|
||||
ExpiresAt: 1710832067 + int64(i),
|
||||
StaleAt: fasttime.Now().Unix() + int64(i),
|
||||
CreatedAt: fasttime.Now().Unix(),
|
||||
}
|
||||
err := table.Set(item.Hash, item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTable_AddField_Delete_Many(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
//runtime.GOMAXPROCS(1)
|
||||
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
{
|
||||
err := table.AddFields("expiresAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
err := table.AddFields("staleAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
err := table.AddFields("url")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var before = time.Now()
|
||||
const from = 0
|
||||
const count = 1_000_000
|
||||
|
||||
for i := from; i < from+count; i++ {
|
||||
var item = &testCachedItem{
|
||||
Hash: "a" + strconv.Itoa(i),
|
||||
}
|
||||
err := table.Delete(item.Hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms", "qps:", int64(float64(count)/costSeconds))
|
||||
|
||||
countLeft, err := table.Count()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("left:", countLeft)
|
||||
}
|
||||
|
||||
func TestTable_DropField(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms")
|
||||
}()
|
||||
|
||||
err := table.DropField("expiresAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
/**func TestTable_DeleteFieldValue(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
err := table.AddField("expiresAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms")
|
||||
}()
|
||||
|
||||
err = table.Delete("a2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testInspectDB(t)
|
||||
}
|
||||
**/
|
||||
|
||||
func TestTable_Inspect(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
err := table.AddFields("expiresAt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testInspectDB(t)
|
||||
}
|
||||
10
internal/utils/kvstore/table_interface.go
Normal file
10
internal/utils/kvstore/table_interface.go
Normal file
@@ -0,0 +1,10 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
type TableInterface interface {
|
||||
Name() string
|
||||
SetNamespace(namespace []byte)
|
||||
SetDB(db *DB)
|
||||
Close() error
|
||||
}
|
||||
403
internal/utils/kvstore/table_test.go
Normal file
403
internal/utils/kvstore/table_test.go
Normal file
@@ -0,0 +1,403 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"math/rand"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTable_Set(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[string]("users", kvstore.NewStringValueEncoder[string]())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
const originValue = "b12345"
|
||||
|
||||
err = table.Set("a", originValue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
value, err := table.Get("a")
|
||||
if err != nil {
|
||||
if kvstore.IsKeyNotFound(err) {
|
||||
t.Log("not found key")
|
||||
return
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("value:", value)
|
||||
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsTrue(originValue == value)
|
||||
}
|
||||
|
||||
func TestTable_Get(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[string]("users", kvstore.NewStringValueEncoder[string]())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
for _, key := range []string{"a", "b", "c"} {
|
||||
value, getErr := table.Get(key)
|
||||
if getErr != nil {
|
||||
if kvstore.IsKeyNotFound(getErr) {
|
||||
t.Log("not found key", key)
|
||||
continue
|
||||
}
|
||||
t.Fatal(getErr)
|
||||
}
|
||||
t.Log(key, "=>", "value:", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTable_Exist(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[string]("users", kvstore.NewStringValueEncoder[string]())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
for _, key := range []string{"a", "b", "c", "12345"} {
|
||||
b, checkErr := table.Exist(key)
|
||||
if checkErr != nil {
|
||||
t.Fatal(checkErr)
|
||||
}
|
||||
t.Log(key, "=>", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTable_Delete(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[string]("users", kvstore.NewStringValueEncoder[string]())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
value, err := table.Get("a123")
|
||||
if err != nil {
|
||||
if !kvstore.IsKeyNotFound(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
t.Log("old value:", value)
|
||||
}
|
||||
|
||||
err = table.Set("a123", "123456")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
{
|
||||
value, err = table.Get("a123")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(value == "123456")
|
||||
}
|
||||
|
||||
err = table.Delete("a123")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
{
|
||||
_, err = table.Get("a123")
|
||||
a.IsTrue(kvstore.IsKeyNotFound(err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTable_Delete_Empty(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[string]("users", kvstore.NewStringValueEncoder[string]())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
{
|
||||
err = table.Delete("a1", "a2", "a3", "a4", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
err = table.Delete()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// set new
|
||||
err = table.Set("a123", "123456")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// delete again
|
||||
{
|
||||
err = table.Delete("a1", "a2", "a3", "a4", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
err = table.Delete()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// read
|
||||
{
|
||||
var value string
|
||||
value, err = table.Get("a123")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(value == "123456")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTable_Count(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
|
||||
var before = time.Now()
|
||||
count, err := table.Count()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("count:", count, "cost:", costSeconds*1000, "ms", "qps:", fmt.Sprintf("%.2fM/s", float64(count)/costSeconds/1_000_000))
|
||||
|
||||
// watch memory usage
|
||||
if testutils.IsSingleTesting() {
|
||||
//time.Sleep(5 * time.Minute)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTable_Truncate(t *testing.T) {
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
var before = time.Now()
|
||||
err := table.Truncate()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
t.Log("cost:", costSeconds*1000, "ms")
|
||||
|
||||
t.Log("===after truncate===")
|
||||
testInspectDB(t)
|
||||
}
|
||||
|
||||
func TestTable_ComposeFieldKey(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
|
||||
var fieldKeyBytes = table.ComposeFieldKey([]byte("Lily"), "username", []byte("lucy"))
|
||||
t.Log(string(fieldKeyBytes))
|
||||
fieldValueBytes, keyValueBytes, err := table.DecodeFieldKey("username", fieldKeyBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("field:", string(fieldValueBytes), "key:", string(keyValueBytes))
|
||||
a.IsTrue(string(fieldValueBytes) == "lucy")
|
||||
a.IsTrue(string(keyValueBytes) == "Lily")
|
||||
}
|
||||
|
||||
func BenchmarkTable_Set(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[uint8]("users", kvstore.NewIntValueEncoder[uint8]())
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
putErr := table.Set(strconv.Itoa(rand.Int()), 1)
|
||||
if putErr != nil {
|
||||
b.Fatal(putErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkTable_Get(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[uint8]("users", kvstore.NewIntValueEncoder[uint8]())
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_, putErr := table.Get(types.String(rand.Int()))
|
||||
if putErr != nil {
|
||||
if kvstore.IsKeyNotFound(putErr) {
|
||||
continue
|
||||
}
|
||||
b.Fatal(putErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**func BenchmarkTable_NextId(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewTable[uint8]("users", kvstore.NewIntValueEncoder[uint8]())
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_, nextErr := table.NextId("a")
|
||||
if nextErr != nil {
|
||||
b.Fatal(nextErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
**/
|
||||
81
internal/utils/kvstore/tx.go
Normal file
81
internal/utils/kvstore/tx.go
Normal file
@@ -0,0 +1,81 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cockroachdb/pebble"
|
||||
)
|
||||
|
||||
type Tx[T any] struct {
|
||||
table *Table[T]
|
||||
readOnly bool
|
||||
|
||||
batch *pebble.Batch
|
||||
}
|
||||
|
||||
func NewTx[T any](table *Table[T], readOnly bool) *Tx[T] {
|
||||
return &Tx[T]{
|
||||
table: table,
|
||||
readOnly: readOnly,
|
||||
batch: table.db.store.rawDB.NewIndexedBatch(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Tx[T]) Set(key string, value T) error {
|
||||
if this.readOnly {
|
||||
return errors.New("can not set value in readonly transaction")
|
||||
}
|
||||
|
||||
if len(key) > KeyMaxLength {
|
||||
return ErrKeyTooLong
|
||||
}
|
||||
|
||||
valueBytes, err := this.table.encoder.Encode(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return this.table.set(this, key, valueBytes, value)
|
||||
}
|
||||
|
||||
func (this *Tx[T]) Get(key string) (value T, err error) {
|
||||
return this.table.get(this, key)
|
||||
}
|
||||
|
||||
func (this *Tx[T]) Delete(key string) error {
|
||||
if this.readOnly {
|
||||
return errors.New("can not delete value in readonly transaction")
|
||||
}
|
||||
|
||||
return this.table.deleteKeys(this, key)
|
||||
}
|
||||
|
||||
func (this *Tx[T]) NewIterator(opt *IteratorOptions) (*pebble.Iterator, error) {
|
||||
return this.batch.NewIter(opt.RawOptions())
|
||||
}
|
||||
|
||||
func (this *Tx[T]) Close() error {
|
||||
return this.batch.Close()
|
||||
}
|
||||
|
||||
func (this *Tx[T]) Commit() (err error) {
|
||||
defer func() {
|
||||
var panicErr = recover()
|
||||
if panicErr != nil {
|
||||
resultErr, ok := panicErr.(error)
|
||||
if ok {
|
||||
err = fmt.Errorf("commit batch failed: %w", resultErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return this.batch.Commit(DefaultWriteOptions)
|
||||
}
|
||||
|
||||
func (this *Tx[T]) Query() *Query[T] {
|
||||
var query = NewQuery[T]()
|
||||
query.SetTx(this)
|
||||
return query
|
||||
}
|
||||
65
internal/utils/kvstore/tx_test.go
Normal file
65
internal/utils/kvstore/tx_test.go
Normal file
@@ -0,0 +1,65 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTable_ReadTx(t *testing.T) {
|
||||
store, err := kvstore.OpenStore("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = store.Close()
|
||||
}()
|
||||
|
||||
db, err := store.NewDB("TEST_DB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
table, err := kvstore.NewCounterTable[uint64]("users_counter")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db.AddTable(table)
|
||||
|
||||
err = table.WriteTx(func(tx *kvstore.Tx[uint64]) error {
|
||||
for i := 0; i < 1000; i++ {
|
||||
var key = fmt.Sprintf("a%03d", i)
|
||||
setErr := tx.Set(key, uint64(i))
|
||||
if setErr != nil {
|
||||
return setErr
|
||||
}
|
||||
|
||||
value, getErr := tx.Get(key)
|
||||
if getErr != nil {
|
||||
return getErr
|
||||
}
|
||||
t.Log("write:", key, "=>", value)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = table.ReadTx(func(tx *kvstore.Tx[uint64]) error {
|
||||
for _, key := range []string{"a100", "a101", "a102"} {
|
||||
value, getErr := tx.Get(key)
|
||||
if getErr != nil {
|
||||
return getErr
|
||||
}
|
||||
t.Log("read:", key, "=>", value)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
50
internal/utils/kvstore/utils.go
Normal file
50
internal/utils/kvstore/utils.go
Normal file
@@ -0,0 +1,50 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var nameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`)
|
||||
|
||||
// IsValidName check if store name or database name or table name is valid
|
||||
func IsValidName(name string) bool {
|
||||
return nameRegexp.MatchString(name)
|
||||
}
|
||||
|
||||
// RemoveStore remove store directory
|
||||
func RemoveStore(path string) error {
|
||||
var errNotStoreDirectory = errors.New("not store directory")
|
||||
|
||||
if strings.HasSuffix(path, StoreSuffix) {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// validate store
|
||||
{
|
||||
_, err = os.Stat(path + "/CURRENT")
|
||||
if err != nil {
|
||||
return errNotStoreDirectory
|
||||
}
|
||||
}
|
||||
{
|
||||
_, err = os.Stat(path + "/LOCK")
|
||||
if err != nil {
|
||||
return errNotStoreDirectory
|
||||
}
|
||||
}
|
||||
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
|
||||
return errNotStoreDirectory
|
||||
}
|
||||
29
internal/utils/kvstore/utils_test.go
Normal file
29
internal/utils/kvstore/utils_test.go
Normal file
@@ -0,0 +1,29 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRemoveDB(t *testing.T) {
|
||||
err := kvstore.RemoveStore(Tea.Root + "/data/stores/test.store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsValidName(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
a.IsTrue(kvstore.IsValidName("a"))
|
||||
a.IsTrue(kvstore.IsValidName("aB"))
|
||||
a.IsTrue(kvstore.IsValidName("aBC1"))
|
||||
a.IsTrue(kvstore.IsValidName("aBC1._-"))
|
||||
a.IsFalse(kvstore.IsValidName(" aBC1._-"))
|
||||
a.IsFalse(kvstore.IsValidName(""))
|
||||
}
|
||||
81
internal/utils/kvstore/value_encode_int.go
Normal file
81
internal/utils/kvstore/value_encode_int.go
Normal file
@@ -0,0 +1,81 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"golang.org/x/exp/constraints"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type IntValueEncoder[T constraints.Integer] struct {
|
||||
}
|
||||
|
||||
func NewIntValueEncoder[T constraints.Integer]() *IntValueEncoder[T] {
|
||||
return &IntValueEncoder[T]{}
|
||||
}
|
||||
|
||||
func (this *IntValueEncoder[T]) Encode(value T) (data []byte, err error) {
|
||||
switch v := any(value).(type) {
|
||||
case int8, int16, int32, int, uint:
|
||||
data = []byte(types.String(v))
|
||||
case int64:
|
||||
data = []byte(strconv.FormatInt(v, 16))
|
||||
case uint8:
|
||||
return []byte{v}, nil
|
||||
case uint16:
|
||||
data = make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(data, v)
|
||||
case uint32:
|
||||
data = make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(data, v)
|
||||
case uint64:
|
||||
data = make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(data, v)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (this *IntValueEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
_ = fieldName
|
||||
return this.Encode(value)
|
||||
}
|
||||
|
||||
func (this *IntValueEncoder[T]) Decode(valueData []byte) (value T, err error) {
|
||||
switch any(value).(type) {
|
||||
case int8:
|
||||
value = T(types.Int8(string(valueData)))
|
||||
case int16:
|
||||
value = T(types.Int16(string(valueData)))
|
||||
case int32:
|
||||
value = T(types.Int32(string(valueData)))
|
||||
case int64:
|
||||
int64Value, _ := strconv.ParseInt(string(valueData), 16, 64)
|
||||
value = T(int64Value)
|
||||
case int:
|
||||
value = T(types.Int(string(valueData)))
|
||||
case uint:
|
||||
value = T(types.Uint(string(valueData)))
|
||||
case uint8:
|
||||
if len(valueData) == 1 {
|
||||
value = T(valueData[0])
|
||||
}
|
||||
case uint16:
|
||||
if len(valueData) == 2 {
|
||||
value = T(binary.BigEndian.Uint16(valueData))
|
||||
}
|
||||
case uint32:
|
||||
if len(valueData) == 4 {
|
||||
value = T(binary.BigEndian.Uint32(valueData))
|
||||
}
|
||||
case uint64:
|
||||
if len(valueData) == 8 {
|
||||
value = T(binary.BigEndian.Uint64(valueData))
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
23
internal/utils/kvstore/value_encoder.go
Normal file
23
internal/utils/kvstore/value_encoder.go
Normal file
@@ -0,0 +1,23 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
type ValueEncoder[T any] interface {
|
||||
Encode(value T) ([]byte, error)
|
||||
EncodeField(value T, fieldName string) ([]byte, error)
|
||||
Decode(valueBytes []byte) (value T, err error)
|
||||
}
|
||||
|
||||
type BaseObjectEncoder[T any] struct {
|
||||
}
|
||||
|
||||
func (this *BaseObjectEncoder[T]) Encode(value T) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
|
||||
func (this *BaseObjectEncoder[T]) Decode(valueData []byte) (value T, err error) {
|
||||
err = json.Unmarshal(valueData, &value)
|
||||
return
|
||||
}
|
||||
29
internal/utils/kvstore/value_encoder_bool.go
Normal file
29
internal/utils/kvstore/value_encoder_bool.go
Normal file
@@ -0,0 +1,29 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
type BoolValueEncoder[T bool] struct {
|
||||
}
|
||||
|
||||
func NewBoolValueEncoder[T bool]() *BoolValueEncoder[T] {
|
||||
return &BoolValueEncoder[T]{}
|
||||
}
|
||||
|
||||
func (this *BoolValueEncoder[T]) Encode(value T) ([]byte, error) {
|
||||
if value {
|
||||
return []byte{1}, nil
|
||||
}
|
||||
return []byte{0}, nil
|
||||
}
|
||||
|
||||
func (this *BoolValueEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
_ = fieldName
|
||||
return this.Encode(value)
|
||||
}
|
||||
|
||||
func (this *BoolValueEncoder[T]) Decode(valueData []byte) (value T, err error) {
|
||||
if len(valueData) == 1 {
|
||||
value = valueData[0] == 1
|
||||
}
|
||||
return
|
||||
}
|
||||
24
internal/utils/kvstore/value_encoder_bytes.go
Normal file
24
internal/utils/kvstore/value_encoder_bytes.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
type BytesValueEncoder[T []byte] struct {
|
||||
}
|
||||
|
||||
func NewBytesValueEncoder[T []byte]() *BytesValueEncoder[T] {
|
||||
return &BytesValueEncoder[T]{}
|
||||
}
|
||||
|
||||
func (this *BytesValueEncoder[T]) Encode(value T) ([]byte, error) {
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (this *BytesValueEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
_ = fieldName
|
||||
return this.Encode(value)
|
||||
}
|
||||
|
||||
func (this *BytesValueEncoder[T]) Decode(valueData []byte) (value T, err error) {
|
||||
value = valueData
|
||||
return
|
||||
}
|
||||
24
internal/utils/kvstore/value_encoder_string.go
Normal file
24
internal/utils/kvstore/value_encoder_string.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore
|
||||
|
||||
type StringValueEncoder[T string] struct {
|
||||
}
|
||||
|
||||
func NewStringValueEncoder[T string]() *StringValueEncoder[T] {
|
||||
return &StringValueEncoder[T]{}
|
||||
}
|
||||
|
||||
func (this *StringValueEncoder[T]) Encode(value T) ([]byte, error) {
|
||||
return []byte(value), nil
|
||||
}
|
||||
|
||||
func (this *StringValueEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
_ = fieldName
|
||||
return this.Encode(value)
|
||||
}
|
||||
|
||||
func (this *StringValueEncoder[T]) Decode(valueData []byte) (value T, err error) {
|
||||
value = T(valueData)
|
||||
return
|
||||
}
|
||||
469
internal/utils/kvstore/value_encoder_test.go
Normal file
469
internal/utils/kvstore/value_encoder_test.go
Normal file
@@ -0,0 +1,469 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package kvstore_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStringValueEncoder_Encode(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var encoder = kvstore.NewStringValueEncoder[string]()
|
||||
data, err := encoder.Encode("abcdefg")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
value, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(value == "abcdefg")
|
||||
}
|
||||
|
||||
func TestIntValueEncoder_Encode(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int8]()
|
||||
data, err := encoder.Encode(1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 1)
|
||||
t.Log("int8", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int8]()
|
||||
data, err := encoder.Encode(-1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == -1)
|
||||
t.Log("int8", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int16]()
|
||||
data, err := encoder.Encode(123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123)
|
||||
t.Log("int16", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int16]()
|
||||
data, err := encoder.Encode(-123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == -123)
|
||||
t.Log("int16", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int32]()
|
||||
data, err := encoder.Encode(123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123)
|
||||
t.Log("int32", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int32]()
|
||||
data, err := encoder.Encode(-123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == -123)
|
||||
t.Log("int32", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int64]()
|
||||
data, err := encoder.Encode(123456)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123456)
|
||||
t.Log("int64", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int64]()
|
||||
data, err := encoder.Encode(1234567890)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 1234567890)
|
||||
t.Log("int64", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int64]()
|
||||
data, err := encoder.Encode(-123456)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == -123456)
|
||||
t.Log("int64", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int]()
|
||||
data, err := encoder.Encode(123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123)
|
||||
t.Log("int", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[int]()
|
||||
data, err := encoder.Encode(-123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == -123)
|
||||
t.Log("int", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[uint]()
|
||||
data, err := encoder.Encode(123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123)
|
||||
t.Log("uint", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[uint8]()
|
||||
data, err := encoder.Encode(97)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 97)
|
||||
t.Log("uint8", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[uint16]()
|
||||
data, err := encoder.Encode(123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123)
|
||||
t.Log("uint16", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[uint32]()
|
||||
data, err := encoder.Encode(123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123)
|
||||
t.Log("uint32", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[uint64]()
|
||||
data, err := encoder.Encode(123)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 123)
|
||||
t.Log("uint64", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
|
||||
{
|
||||
var encoder = kvstore.NewIntValueEncoder[uint64]()
|
||||
data, err := encoder.Encode(1234567890)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(v == 1234567890)
|
||||
t.Log("uint64", string(data), "=>", data, "=>", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesValueEncoder_Encode(t *testing.T) {
|
||||
var encoder = kvstore.NewBytesValueEncoder[[]byte]()
|
||||
{
|
||||
data, err := encoder.Encode(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(data, "=>", value)
|
||||
}
|
||||
|
||||
{
|
||||
data, err := encoder.Encode([]byte("ABC"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(data, "=>", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesValueEncoder_Bool(t *testing.T) {
|
||||
var encoder = kvstore.NewBoolValueEncoder[bool]()
|
||||
{
|
||||
data, err := encoder.Encode(true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(data, "=>", value)
|
||||
}
|
||||
|
||||
{
|
||||
data, err := encoder.Encode(false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(data, "=>", value)
|
||||
}
|
||||
|
||||
{
|
||||
value, err := encoder.Decode(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("nil", "=>", value)
|
||||
}
|
||||
|
||||
{
|
||||
value, err := encoder.Decode([]byte{1, 2, 3, 4})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("nil", "=>", value)
|
||||
}
|
||||
}
|
||||
|
||||
type objectType struct {
|
||||
Name string `json:"1"`
|
||||
Age int `json:"2"`
|
||||
}
|
||||
|
||||
type objectTypeEncoder[T objectType] struct {
|
||||
kvstore.BaseObjectEncoder[T]
|
||||
}
|
||||
|
||||
func (this *objectTypeEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestBaseObjectEncoder_Encode(t *testing.T) {
|
||||
var encoder = &objectTypeEncoder[objectType]{}
|
||||
|
||||
{
|
||||
data, err := encoder.Encode(objectType{
|
||||
Name: "lily",
|
||||
Age: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("encoded:", string(data))
|
||||
}
|
||||
|
||||
{
|
||||
value, err := encoder.Decode([]byte(`{"1":"lily","2":20}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("decoded: %+v", value)
|
||||
}
|
||||
}
|
||||
|
||||
type objectType2 struct {
|
||||
Name string `json:"1"`
|
||||
Age int `json:"2"`
|
||||
}
|
||||
|
||||
type objectTypeEncoder2[T interface{ *objectType2 }] struct {
|
||||
kvstore.BaseObjectEncoder[T]
|
||||
}
|
||||
|
||||
func (this *objectTypeEncoder2[T]) EncodeField(value T, fieldName string) ([]byte, error) {
|
||||
switch fieldName {
|
||||
case "Name":
|
||||
return []byte(any(value).(*objectType2).Name), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestBaseObjectEncoder_Encode2(t *testing.T) {
|
||||
var encoder = &objectTypeEncoder2[*objectType2]{}
|
||||
|
||||
{
|
||||
data, err := encoder.Encode(&objectType2{
|
||||
Name: "lily",
|
||||
Age: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("encoded:", string(data))
|
||||
}
|
||||
|
||||
{
|
||||
value, err := encoder.Decode([]byte(`{"1":"lily","2":20}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("decoded: %+v", value)
|
||||
}
|
||||
|
||||
{
|
||||
field, err := encoder.EncodeField(&objectType2{
|
||||
Name: "lily",
|
||||
Age: 20,
|
||||
}, "Name")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("encoded field:", string(field))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStringValueEncoder_Encode(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var encoder = kvstore.NewStringValueEncoder[string]()
|
||||
data, err := encoder.Encode("1234567890")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
_ = v
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkIntValueEncoder_Encode(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var encoder = kvstore.NewIntValueEncoder[int64]()
|
||||
data, err := encoder.Encode(1234567890)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
_ = v
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUIntValueEncoder_Encode(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var encoder = kvstore.NewIntValueEncoder[uint64]()
|
||||
data, err := encoder.Encode(1234567890)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
v, err := encoder.Decode(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
_ = v
|
||||
}
|
||||
}
|
||||
15
internal/utils/minifiers/minify.go
Normal file
15
internal/utils/minifiers/minify.go
Normal file
@@ -0,0 +1,15 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build !plus
|
||||
|
||||
package minifiers
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// MinifyResponse minify response body
|
||||
func MinifyResponse(config *serverconfigs.HTTPPageOptimizationConfig, url string, resp *http.Response) error {
|
||||
// stub
|
||||
return nil
|
||||
}
|
||||
@@ -115,6 +115,12 @@ func (this *ServiceManager) installSystemdService(systemd, exePath string, args
|
||||
var shortName = teaconst.SystemdServiceName
|
||||
var longName = "GoEdge Node" // TODO 将来可以修改
|
||||
|
||||
var startCmd = exePath + " daemon"
|
||||
bashPath, _ := executils.LookPath("bash")
|
||||
if len(bashPath) > 0 {
|
||||
startCmd = bashPath + " -c \"" + startCmd + "\""
|
||||
}
|
||||
|
||||
var desc = `### BEGIN INIT INFO
|
||||
# Provides: ` + shortName + `
|
||||
# Required-Start: $all
|
||||
@@ -133,7 +139,7 @@ After=network-online.target
|
||||
Type=simple
|
||||
Restart=always
|
||||
RestartSec=1s
|
||||
ExecStart=` + exePath + ` daemon
|
||||
ExecStart=` + startCmd + `
|
||||
ExecStop=` + exePath + ` stop
|
||||
ExecReload=` + exePath + ` reload
|
||||
|
||||
|
||||
@@ -55,3 +55,9 @@ func StartMemoryStats(t *testing.T, callbacks ...func()) {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func ReadMemoryStat() *runtime.MemStats {
|
||||
var stat = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat)
|
||||
return stat
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user