增加KV存储测试用例

build/build-all.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
 ./build.sh linux amd64
-./build.sh linux 386
+#./build.sh linux 386
 ./build.sh linux arm64
-./build.sh linux mips64
-./build.sh linux mips64le
-./build.sh darwin amd64
-./build.sh darwin arm64
+#./build.sh linux mips64
+#./build.sh linux mips64le
+#./build.sh darwin amd64
+#./build.sh darwin arm64

build/test.sh

@@ -6,4 +6,9 @@ if [ -z "$TAG" ]; then
 	TAG="community"
 fi
 
-go test -v ../... -tags=${TAG}
+# stop node
+go run -tags=${TAG}  ../cmd/edge-node/main.go stop
+
+# reference: https://pkg.go.dev/cmd/go/internal/test
+go clean -testcache
+go test -timeout 10s -tags="${TAG}" -cover ../...

cmd/edge-node/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/types"
 	"github.com/iwind/gosock/pkg/gosock"
+	"gopkg.in/yaml.v3"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -30,7 +31,7 @@ func main() {
 	var app = apps.NewAppCmd().
 		Version(teaconst.Version).
 		Product(teaconst.ProductName).
-		Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog|uninstall]").
+		Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|config|pprof|accesslog|uninstall]").
 		Usage(teaconst.ProcessName + " [trackers|goman|conns|gc|bandwidth|disk|cache.garbage]").
 		Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove|ip.close] IP")
 
@@ -524,6 +525,41 @@ func main() {
 			fmt.Println("[ERROR]" + params.GetString("error"))
 		}
 	})
+	app.On("config", func() {
+		var configString = os.Args[len(os.Args)-1]
+		if configString == "config" {
+			fmt.Println("Usage: edge-node config '\nrpc.endpoints: [\"...\"]\nnodeId: \"...\"\nsecret: \"...\"\n'")
+			return
+		}
+
+		var config = &configs.APIConfig{}
+		err := yaml.Unmarshal([]byte(configString), config)
+		if err != nil {
+			fmt.Println("[ERROR]decode config failed: " + err.Error())
+			return
+		}
+
+		err = config.Init()
+		if err != nil {
+			fmt.Println("[ERROR]validate config failed: " + err.Error())
+			return
+		}
+
+		// marshal again
+		configYAML, err := yaml.Marshal(config)
+		if err != nil {
+			fmt.Println("[ERROR]encode config failed: " + err.Error())
+			return
+		}
+
+		err = os.WriteFile(Tea.Root + "/configs/api_node.yaml", configYAML, 0666)
+		if err != nil {
+			fmt.Println("[ERROR]write config failed: " + err.Error())
+			return
+		}
+
+		fmt.Println("success")
+	})
 	app.Run(func() {
 		var node = nodes.NewNode()
 		node.Start()

go.mod

@@ -1,23 +1,25 @@
 module github.com/TeaOSLab/EdgeNode
 
-go 1.18
+go 1.21
 
 replace (
 	github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
+	github.com/dchest/captcha => github.com/iwind/captcha v0.0.0-20231130092438-ae985686ed84
 	github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
 	github.com/google/nftables => github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4
 )
 
 require (
 	github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
-	github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible
+	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
 	github.com/andybalholm/brotli v1.0.5
 	github.com/aws/aws-sdk-go v1.44.279
-	github.com/baidubce/bce-sdk-go v0.9.153
+	github.com/baidubce/bce-sdk-go v0.9.170
 	github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670
 	github.com/cespare/xxhash v1.1.0
-	github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
-	github.com/fsnotify/fsnotify v1.6.0
+	github.com/cespare/xxhash/v2 v2.2.0
+	github.com/dchest/captcha v0.0.0-00010101000000-000000000000
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.1.0
@@ -25,58 +27,81 @@ require (
 	github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d
 	github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
 	github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
-	github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4
-	github.com/klauspost/compress v1.17.2
+	github.com/iwind/gowebp v0.0.0-20240109104518-489f3429f5c5
+	github.com/klauspost/compress v1.17.7
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/mdlayher/netlink v1.7.1
 	github.com/miekg/dns v1.1.43
 	github.com/mssola/useragent v1.0.0
 	github.com/pires/go-proxyproto v0.6.1
 	github.com/qiniu/go-sdk/v7 v7.16.0
-	github.com/quic-go/quic-go v0.39.2
+	github.com/quic-go/quic-go v0.42.0
 	github.com/shirou/gopsutil/v3 v3.22.2
+	github.com/tdewolff/minify/v2 v2.20.19
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.41
+	golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81
 	golang.org/x/image v0.13.0
-	golang.org/x/net v0.17.0
-	golang.org/x/sys v0.13.0
-	google.golang.org/grpc v1.59.0
-	google.golang.org/protobuf v1.31.0
+	golang.org/x/net v0.22.0
+	golang.org/x/sys v0.18.0
+	google.golang.org/grpc v1.62.1
+	google.golang.org/protobuf v1.33.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/DataDog/zstd v1.5.5 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/clbanning/mxj v1.8.4 // indirect
+	github.com/cockroachdb/errors v1.11.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v1.1.0 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.2.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/flatbuffers v24.3.7+incompatible // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
-	github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
+	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.0.0 // indirect
 	github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mdlayher/socket v0.4.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/mozillazg/go-httpheader v0.2.1 // indirect
-	github.com/onsi/ginkgo/v2 v2.13.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.16.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/client_golang v1.19.0 // indirect
+	github.com/prometheus/client_model v0.6.0 // indirect
+	github.com/prometheus/common v0.51.0 // indirect
+	github.com/prometheus/procfs v0.13.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
-	github.com/tdewolff/minify/v2 v2.12.7 // indirect
-	github.com/tdewolff/parse/v2 v2.6.6 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/tdewolff/parse/v2 v2.7.12 // indirect
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
 	github.com/tklauser/numcpus v0.3.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.uber.org/mock v0.3.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/sync v0.4.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.uber.org/mock v0.4.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.19.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
 )

go.sum

@@ -1,34 +1,63 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
+github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
-github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis=
-github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/aws/aws-sdk-go v1.44.279 h1:g23dxnYjIiPlQo0gIKNR0zVPsSvo1bj5frWln+5sfhk=
 github.com/aws/aws-sdk-go v1.44.279/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/baidubce/bce-sdk-go v0.9.153 h1:h5l2EXehe4C4/bdlAPBaULrbnEDgIu5HOYgniN7bjGM=
-github.com/baidubce/bce-sdk-go v0.9.153/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
+github.com/baidubce/bce-sdk-go v0.9.170 h1:vAr7COuhu6SEf+8f77DVRji45x7TVZtY5kbu9sX7q8g=
+github.com/baidubce/bce-sdk-go v0.9.170/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
 github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
 github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
+github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
+github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4=
+github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
+github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
+github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
-github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
+github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -41,34 +70,64 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
+github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/flatbuffers v24.3.7+incompatible h1:BxGUkIQnOciBu33bd5BdvqY8Qvo0O/GR4SPhh7x9Ed0=
+github.com/google/flatbuffers v24.3.7+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
-github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0=
-github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q=
+github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible h1:XRAk4HBDLCYEdPLWtKf5iZhOi7lfx17aY0oSO9+mcg8=
 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
 github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d h1:XnTIj781NdSipts60fVqbgZorVAKVSRaA6nqVNfBQ1g=
 github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
+github.com/iwind/captcha v0.0.0-20231130092438-ae985686ed84 h1:/RtK8t22a/YFkBWiEwxS+JWcDmxAKsu+r+p00c36K0Q=
+github.com/iwind/captcha v0.0.0-20231130092438-ae985686ed84/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
 github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
 github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
 github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
 github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
 github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
 github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
-github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4 h1:eyymORsZg0tZ0niyolYF4nao4sdNUI+Ll40s96tKHBY=
-github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4/go.mod h1:AYyXDhbbD7q9N6rJff2jrE7pGupaiyvtv3YeyIAQLXk=
+github.com/iwind/gowebp v0.0.0-20240109104518-489f3429f5c5 h1:tA0HEDQJ/FM847wc7kVpSgkTfMF1LervEmd2UZQr3Po=
+github.com/iwind/gowebp v0.0.0-20240109104518-489f3429f5c5/go.mod h1:AYyXDhbbD7q9N6rJff2jrE7pGupaiyvtv3YeyIAQLXk=
 github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4 h1:RPAH9Sj9l/20zH5zU5/iJGszfwPq6eLjoiC/n/asulA=
 github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4/go.mod h1:7OLL+86wZKfBnAJxNxmdcZ0ebbgdp/A28fcagx9oJqA=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -77,14 +136,21 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk=
 github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -92,9 +158,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
 github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mdlayher/netlink v1.7.1 h1:FdUaT/e33HjEXagwELR8R3/KL1Fq5x3G5jgHLp/BTmg=
 github.com/mdlayher/netlink v1.7.1/go.mod h1:nKO5CSjE/DJjVhk/TNp6vCE1ktVxEA8VEh8drhZzxsQ=
 github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw=
@@ -103,53 +170,76 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
 github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
 github.com/mssola/useragent v1.0.0 h1:WRlDpXyxHDNfvZaPEut5Biveq86Ze4o4EMffyMxmH5o=
 github.com/mssola/useragent v1.0.0/go.mod h1:hz9Cqz4RXusgg1EdI4Al0INR62kP7aPSRNHnpU+b85Y=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
-github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
-github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM=
+github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
+github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
 github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
 github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
+github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
+github.com/prometheus/common v0.51.0 h1:vT5R9NAlW4V6k8Wruk7ikrHaHRsrPbduM/cKTOdQM/k=
+github.com/prometheus/common v0.51.0/go.mod h1:wHFBCEVWVmHMUpg7pYcOm2QUR/ocQdYSJVQJKnHc3xQ=
+github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o=
+github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
 github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=
 github.com/qiniu/go-sdk/v7 v7.16.0 h1:Jt4YOMLuaDfgb/KdVg0O1fYLpv5MDkYe/zV+Ri7gWRs=
 github.com/qiniu/go-sdk/v7 v7.16.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w=
 github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
-github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
-github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.39.2 h1:hmwAf8zAHlvan0Y5PXxeeBFZEW17IW99sXLry8I2kjk=
-github.com/quic-go/quic-go v0.39.2/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
+github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
+github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
 github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/tdewolff/minify/v2 v2.12.7 h1:pBzz2tAfz5VghOXiQIsSta6srhmTeinQPjRDHWoumCA=
-github.com/tdewolff/minify/v2 v2.12.7/go.mod h1:ZRKTheiOGyLSK8hOZWWv+YoJAECzDivNgAlVYDHp/Ws=
-github.com/tdewolff/parse/v2 v2.6.6 h1:Yld+0CrKUJaCV78DL1G2nk3C9lKrxyRTux5aaK/AkDo=
-github.com/tdewolff/parse/v2 v2.6.6/go.mod h1:woz0cgbLwFdtbjJu8PIKxhW05KplTFQkOdX78o+Jgrs=
-github.com/tdewolff/test v1.0.7/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
-github.com/tdewolff/test v1.0.9 h1:SswqJCmeN4B+9gEAi/5uqT0qpi1y2/2O47V/1hhGZT0=
-github.com/tdewolff/test v1.0.9/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/tdewolff/minify/v2 v2.20.19 h1:tX0SR0LUrIqGoLjXnkIzRSIbKJ7PaNnSENLD4CyH6Xo=
+github.com/tdewolff/minify/v2 v2.20.19/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM=
+github.com/tdewolff/parse/v2 v2.7.12 h1:tgavkHc2ZDEQVKy1oWxwIyh5bP4F5fEh/JmBwPP/3LQ=
+github.com/tdewolff/parse/v2 v2.7.12/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
+github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
+github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
+github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.41 h1:iU0Li/Np78H4SBna0ECQoF3mpgi6ImLXU+doGzPFXGc=
@@ -159,42 +249,71 @@ github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ
 github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
 github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
+github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
-go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f h1:3CW0unweImhOzd5FmYuRsD4Y4oQFKZIjAnKbjV4WIrw=
+golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
+golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc=
+golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
 golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
 golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
-golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -204,11 +323,10 @@ golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -218,36 +336,65 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
-golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
+golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
+golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
-google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
-google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
+google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/caches/errors.go

@@ -34,16 +34,14 @@ func CanIgnoreErr(err error) bool {
 	if err == nil {
 		return true
 	}
-	if err == ErrFileIsWriting ||
-		err == ErrEntityTooLarge ||
-		err == ErrWritingUnavailable ||
-		err == ErrWritingQueueFull ||
-		err == ErrServerIsBusy {
+	if errors.Is(err, ErrFileIsWriting) ||
+		errors.Is(err, ErrEntityTooLarge) ||
+		errors.Is(err, ErrWritingUnavailable) ||
+		errors.Is(err, ErrWritingQueueFull) ||
+		errors.Is(err, ErrServerIsBusy) {
 		return true
 	}
-	_, ok := err.(*CapacityError)
-	if ok {
-		return true
-	}
-	return false
+
+	var capacityErr *CapacityError
+	return errors.As(err, &capacityErr)
 }

internal/caches/errros_test.go

@@ -1,16 +1,24 @@
 // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 
-package caches
+package caches_test
 
 import (
+	"errors"
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/iwind/TeaGo/assert"
 	"testing"
 )
 
 func TestCanIgnoreErr(t *testing.T) {
-	a := assert.NewAssertion(t)
+	var a = assert.NewAssertion(t)
 
-	a.IsTrue(CanIgnoreErr(ErrFileIsWriting))
-	a.IsTrue(CanIgnoreErr(NewCapacityError("over capcity")))
-	a.IsFalse(CanIgnoreErr(ErrNotFound))
+	a.IsTrue(caches.CanIgnoreErr(caches.ErrFileIsWriting))
+	a.IsTrue(caches.CanIgnoreErr(fmt.Errorf("error: %w", caches.ErrFileIsWriting)))
+	a.IsTrue(errors.Is(fmt.Errorf("error: %w", caches.ErrFileIsWriting), caches.ErrFileIsWriting))
+	a.IsTrue(errors.Is(caches.ErrFileIsWriting, caches.ErrFileIsWriting))
+	a.IsTrue(caches.CanIgnoreErr(caches.NewCapacityError("over capacity")))
+	a.IsTrue(caches.CanIgnoreErr(fmt.Errorf("error: %w", caches.NewCapacityError("over capacity"))))
+	a.IsFalse(caches.CanIgnoreErr(caches.ErrNotFound))
+	a.IsFalse(caches.CanIgnoreErr(errors.New("test error")))
 }

internal/caches/item.go

@@ -19,20 +19,21 @@ func currentWeek() int32 {
 }
 
 type Item struct {
-	Type       ItemType `json:"type"`
-	Key        string   `json:"key"`
-	ExpiredAt  int64    `json:"expiredAt"`
-	StaleAt    int64    `json:"staleAt"`
-	HeaderSize int64    `json:"headerSize"`
-	BodySize   int64    `json:"bodySize"`
-	MetaSize   int64    `json:"metaSize"`
-	Host       string   `json:"host"`     // 主机名
-	ServerId   int64    `json:"serverId"` // 服务ID
-	Week       int32    `json:"week"`
+	Type       ItemType `json:"-"`
+	Key        string   `json:"1,omitempty"`
+	ExpiresAt  int64    `json:"2,omitempty"`
+	StaleAt    int64    `json:"3,omitempty"`
+	HeaderSize int64    `json:"-"`
+	BodySize   int64    `json:"4,omitempty"`
+	MetaSize   int64    `json:"-"`
+	Host       string   `json:"-"`           // 主机名
+	ServerId   int64    `json:"5,omitempty"` // 服务ID
+	Week       int32    `json:"-"`
+	CreatedAt  int64    `json:"6,omitempty"`
 }
 
 func (this *Item) IsExpired() bool {
-	return this.ExpiredAt < fasttime.Now().Unix()
+	return this.ExpiresAt < fasttime.Now().Unix()
 }
 
 func (this *Item) TotalSize() int64 {

internal/caches/item_test.go

@@ -3,7 +3,10 @@
 package caches_test
 
 import (
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
@@ -12,13 +15,44 @@ import (
 	"time"
 )
 
+func TestItem_Marshal(t *testing.T) {
+	{
+		var item = &caches.Item{}
+		data, err := json.Marshal(item)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(string(data))
+	}
+
+	{
+		var item = &caches.Item{
+			Type:       caches.ItemTypeFile,
+			Key:        "https://example.com/index.html",
+			ExpiresAt:  fasttime.Now().Unix(),
+			HeaderSize: 1 << 10,
+			BodySize:   1 << 20,
+			MetaSize:   256,
+		}
+		data, err := json.Marshal(item)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(string(data))
+	}
+}
+
 func TestItems_Memory(t *testing.T) {
 	var stat = &runtime.MemStats{}
 	runtime.ReadMemStats(stat)
 	var memory1 = stat.HeapInuse
 
 	var items = []*caches.Item{}
-	for i := 0; i < 10_000_000; i++ {
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 10_000_000
+	}
+	for i := 0; i < count; i++ {
 		items = append(items, &caches.Item{
 			Key: types.String(i),
 		})
@@ -33,7 +67,9 @@ func TestItems_Memory(t *testing.T) {
 	var memory3 = stat.HeapInuse
 	t.Log(memory2, memory3, (memory3-memory2)/1024/1024, "M")
 
-	time.Sleep(1 * time.Second)
+	if testutils.IsSingleTesting() {
+		time.Sleep(1 * time.Second)
+	}
 }
 
 func TestItems_Memory2(t *testing.T) {
@@ -42,7 +78,12 @@ func TestItems_Memory2(t *testing.T) {
 	var memory1 = stat.HeapInuse
 
 	var items = map[int32]map[string]zero.Zero{}
-	for i := 0; i < 10_000_000; i++ {
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 10_000_000
+	}
+
+	for i := 0; i < count; i++ {
 		var week = int32((time.Now().Unix() - int64(86400*rands.Int(0, 300))) / (86400 * 7))
 		m, ok := items[week]
 		if !ok {
@@ -57,7 +98,9 @@ func TestItems_Memory2(t *testing.T) {
 
 	t.Log(memory1, memory2, (memory2-memory1)/1024/1024, "M")
 
-	time.Sleep(1 * time.Second)
+	if testutils.IsSingleTesting() {
+		time.Sleep(1 * time.Second)
+	}
 	for w, i := range items {
 		t.Log(w, len(i))
 	}

internal/caches/list_file_db_sqlite.go

@@ -20,13 +20,13 @@ import (
 	"time"
 )
 
-type FileListDB struct {
+type SQLiteFileListDB struct {
 	dbPath string
 
 	readDB  *dbs.DB
 	writeDB *dbs.DB
 
-	hashMap *FileListHashMap
+	hashMap *SQLiteFileListHashMap
 
 	itemsTableName string
 
@@ -53,13 +53,13 @@ type FileListDB struct {
 	listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
 }
 
-func NewFileListDB() *FileListDB {
-	return &FileListDB{
-		hashMap: NewFileListHashMap(),
+func NewSQLiteFileListDB() *SQLiteFileListDB {
+	return &SQLiteFileListDB{
+		hashMap: NewSQLiteFileListHashMap(),
 	}
 }
 
-func (this *FileListDB) Open(dbPath string) error {
+func (this *SQLiteFileListDB) Open(dbPath string) error {
 	this.dbPath = dbPath
 
 	// 动态调整Cache值
@@ -119,7 +119,7 @@ func (this *FileListDB) Open(dbPath string) error {
 	return nil
 }
 
-func (this *FileListDB) Init() error {
+func (this *SQLiteFileListDB) Init() error {
 	this.itemsTableName = "cacheItems"
 
 	// 创建
@@ -184,11 +184,11 @@ func (this *FileListDB) Init() error {
 	return nil
 }
 
-func (this *FileListDB) IsReady() bool {
+func (this *SQLiteFileListDB) IsReady() bool {
 	return this.isReady
 }
 
-func (this *FileListDB) Total() (int64, error) {
+func (this *SQLiteFileListDB) Total() (int64, error) {
 	// 读取总数量
 	var row = this.readDB.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
 	if row.Err() != nil {
@@ -199,14 +199,14 @@ func (this *FileListDB) Total() (int64, error) {
 	return total, err
 }
 
-func (this *FileListDB) AddSync(hash string, item *Item) error {
+func (this *SQLiteFileListDB) AddSync(hash string, item *Item) error {
 	this.hashMap.Add(hash)
 
 	if item.StaleAt == 0 {
-		item.StaleAt = item.ExpiredAt
+		item.StaleAt = item.ExpiresAt
 	}
 
-	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
+	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiresAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
 	if err != nil {
 		return this.WrapError(err)
 	}
@@ -214,7 +214,7 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
 	return nil
 }
 
-func (this *FileListDB) DeleteSync(hash string) error {
+func (this *SQLiteFileListDB) DeleteSync(hash string) error {
 	this.hashMap.Delete(hash)
 
 	_, err := this.deleteByHashStmt.Exec(hash)
@@ -224,7 +224,7 @@ func (this *FileListDB) DeleteSync(hash string) error {
 	return nil
 }
 
-func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err error) {
+func (this *SQLiteFileListDB) ListExpiredItems(count int) (hashList []string, err error) {
 	if !this.isReady {
 		return nil, nil
 	}
@@ -252,7 +252,7 @@ func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err erro
 	return hashList, nil
 }
 
-func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
+func (this *SQLiteFileListDB) ListLFUItems(count int) (hashList []string, err error) {
 	if !this.isReady {
 		return nil, nil
 	}
@@ -280,7 +280,7 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
 	return hashList, nil
 }
 
-func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
+func (this *SQLiteFileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
 	rows, err := this.selectHashListStmt.Query(lastId)
 	if err != nil {
 		return nil, 0, err
@@ -301,12 +301,12 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
 	return
 }
 
-func (this *FileListDB) IncreaseHitAsync(hash string) error {
+func (this *SQLiteFileListDB) IncreaseHitAsync(hash string) error {
 	// do nothing
 	return nil
 }
 
-func (this *FileListDB) CleanPrefix(prefix string) error {
+func (this *SQLiteFileListDB) CleanPrefix(prefix string) error {
 	if !this.isReady {
 		return nil
 	}
@@ -327,7 +327,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
 	}
 }
 
-func (this *FileListDB) CleanMatchKey(key string) error {
+func (this *SQLiteFileListDB) CleanMatchKey(key string) error {
 	if !this.isReady {
 		return nil
 	}
@@ -372,7 +372,7 @@ func (this *FileListDB) CleanMatchKey(key string) error {
 	return nil
 }
 
-func (this *FileListDB) CleanMatchPrefix(prefix string) error {
+func (this *SQLiteFileListDB) CleanMatchPrefix(prefix string) error {
 	if !this.isReady {
 		return nil
 	}
@@ -404,7 +404,7 @@ func (this *FileListDB) CleanMatchPrefix(prefix string) error {
 	return err
 }
 
-func (this *FileListDB) CleanAll() error {
+func (this *SQLiteFileListDB) CleanAll() error {
 	if !this.isReady {
 		return nil
 	}
@@ -419,7 +419,7 @@ func (this *FileListDB) CleanAll() error {
 	return nil
 }
 
-func (this *FileListDB) Close() error {
+func (this *SQLiteFileListDB) Close() error {
 	if this.isClosed {
 		return nil
 	}
@@ -477,19 +477,19 @@ func (this *FileListDB) Close() error {
 	return errors.New("close database failed: " + strings.Join(errStrings, ", "))
 }
 
-func (this *FileListDB) WrapError(err error) error {
+func (this *SQLiteFileListDB) WrapError(err error) error {
 	if err == nil {
 		return nil
 	}
 	return fmt.Errorf("%w (file: %s)", err, this.dbPath)
 }
 
-func (this *FileListDB) HashMapIsLoaded() bool {
+func (this *SQLiteFileListDB) HashMapIsLoaded() bool {
 	return this.hashMapIsLoaded
 }
 
 // 初始化
-func (this *FileListDB) initTables(times int) error {
+func (this *SQLiteFileListDB) initTables(times int) error {
 	{
 		// expiredAt - 过期时间，用来判断有无过期
 		// staleAt - 过时缓存最大时间，用来清理缓存
@@ -553,7 +553,7 @@ ON "` + this.itemsTableName + `" (
 	return nil
 }
 
-func (this *FileListDB) listOlderItems(count int) (hashList []string, err error) {
+func (this *SQLiteFileListDB) listOlderItems(count int) (hashList []string, err error) {
 	rows, err := this.listOlderItemsStmt.Query(count)
 	if err != nil {
 		return nil, err
@@ -574,7 +574,7 @@ func (this *FileListDB) listOlderItems(count int) (hashList []string, err error)
 	return hashList, nil
 }
 
-func (this *FileListDB) shouldRecover() bool {
+func (this *SQLiteFileListDB) shouldRecover() bool {
 	result, err := this.writeDB.Query("pragma integrity_check;")
 	if err != nil {
 		logs.Println(result)
@@ -592,14 +592,14 @@ func (this *FileListDB) shouldRecover() bool {
 }
 
 // 删除数据库文件
-func (this *FileListDB) deleteDB() {
+func (this *SQLiteFileListDB) deleteDB() {
 	_ = os.Remove(this.dbPath)
 	_ = os.Remove(this.dbPath + "-shm")
 	_ = os.Remove(this.dbPath + "-wal")
 }
 
 // 加载Hash列表
-func (this *FileListDB) loadHashMap() {
+func (this *SQLiteFileListDB) loadHashMap() {
 	this.hashMapIsLoaded = false
 
 	err := this.hashMap.Load(this)

internal/caches/list_file_db_sqlite_test.go

@@ -18,7 +18,7 @@ func TestFileListDB_ListLFUItems(t *testing.T) {
 		return
 	}
 
-	var db = caches.NewFileListDB()
+	var db = caches.NewSQLiteFileListDB()
 
 	defer func() {
 		_ = db.Close()
@@ -46,7 +46,7 @@ func TestFileListDB_CleanMatchKey(t *testing.T) {
 		return
 	}
 
-	var db = caches.NewFileListDB()
+	var db = caches.NewSQLiteFileListDB()
 
 	defer func() {
 		_ = db.Close()
@@ -78,7 +78,7 @@ func TestFileListDB_CleanMatchPrefix(t *testing.T) {
 		return
 	}
 
-	var db = caches.NewFileListDB()
+	var db = caches.NewSQLiteFileListDB()
 
 	defer func() {
 		_ = db.Close()
@@ -110,7 +110,7 @@ func TestFileListDB_Memory(t *testing.T) {
 		return
 	}
 
-	var db = caches.NewFileListDB()
+	var db = caches.NewSQLiteFileListDB()
 
 	defer func() {
 		_ = db.Close()

internal/caches/list_file_hash_map_sqlite.go

@@ -17,8 +17,8 @@ var bigIntPool = sync.Pool{
 	},
 }
 
-// FileListHashMap 文件Hash列表
-type FileListHashMap struct {
+// SQLiteFileListHashMap 文件Hash列表
+type SQLiteFileListHashMap struct {
 	m []map[uint64]zero.Zero
 
 	lockers []*sync.RWMutex
@@ -27,7 +27,7 @@ type FileListHashMap struct {
 	isReady     bool
 }
 
-func NewFileListHashMap() *FileListHashMap {
+func NewSQLiteFileListHashMap() *SQLiteFileListHashMap {
 	var m = make([]map[uint64]zero.Zero, HashMapSharding)
 	var lockers = make([]*sync.RWMutex, HashMapSharding)
 
@@ -36,7 +36,7 @@ func NewFileListHashMap() *FileListHashMap {
 		lockers[i] = &sync.RWMutex{}
 	}
 
-	return &FileListHashMap{
+	return &SQLiteFileListHashMap{
 		m:           m,
 		lockers:     lockers,
 		isAvailable: false,
@@ -44,7 +44,7 @@ func NewFileListHashMap() *FileListHashMap {
 	}
 }
 
-func (this *FileListHashMap) Load(db *FileListDB) error {
+func (this *SQLiteFileListHashMap) Load(db *SQLiteFileListDB) error {
 	// 如果系统内存过小，我们不缓存
 	if utils.SystemMemoryGB() < 3 {
 		return nil
@@ -75,7 +75,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
 	return nil
 }
 
-func (this *FileListHashMap) Add(hash string) {
+func (this *SQLiteFileListHashMap) Add(hash string) {
 	if !this.isAvailable {
 		return
 	}
@@ -87,7 +87,7 @@ func (this *FileListHashMap) Add(hash string) {
 	this.lockers[index].Unlock()
 }
 
-func (this *FileListHashMap) AddHashes(hashes []string) {
+func (this *SQLiteFileListHashMap) AddHashes(hashes []string) {
 	if !this.isAvailable {
 		return
 	}
@@ -100,7 +100,7 @@ func (this *FileListHashMap) AddHashes(hashes []string) {
 	}
 }
 
-func (this *FileListHashMap) Delete(hash string) {
+func (this *SQLiteFileListHashMap) Delete(hash string) {
 	if !this.isAvailable {
 		return
 	}
@@ -111,7 +111,7 @@ func (this *FileListHashMap) Delete(hash string) {
 	this.lockers[index].Unlock()
 }
 
-func (this *FileListHashMap) Exist(hash string) bool {
+func (this *SQLiteFileListHashMap) Exist(hash string) bool {
 	if !this.isAvailable {
 		return true
 	}
@@ -128,7 +128,7 @@ func (this *FileListHashMap) Exist(hash string) bool {
 	return ok
 }
 
-func (this *FileListHashMap) Clean() {
+func (this *SQLiteFileListHashMap) Clean() {
 	for i := 0; i < HashMapSharding; i++ {
 		this.lockers[i].Lock()
 	}
@@ -143,11 +143,11 @@ func (this *FileListHashMap) Clean() {
 	}
 }
 
-func (this *FileListHashMap) IsReady() bool {
+func (this *SQLiteFileListHashMap) IsReady() bool {
 	return this.isReady
 }
 
-func (this *FileListHashMap) Len() int {
+func (this *SQLiteFileListHashMap) Len() int {
 	for i := 0; i < HashMapSharding; i++ {
 		this.lockers[i].Lock()
 	}
@@ -164,15 +164,15 @@ func (this *FileListHashMap) Len() int {
 	return count
 }
 
-func (this *FileListHashMap) SetIsAvailable(isAvailable bool) {
+func (this *SQLiteFileListHashMap) SetIsAvailable(isAvailable bool) {
 	this.isAvailable = isAvailable
 }
 
-func (this *FileListHashMap) SetIsReady(isReady bool) {
+func (this *SQLiteFileListHashMap) SetIsReady(isReady bool) {
 	this.isReady = isReady
 }
 
-func (this *FileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
+func (this *SQLiteFileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
 	var bigInt = bigIntPool.Get().(*big.Int)
 	bigInt.SetString(hash, 16)
 	hashInt = bigInt.Uint64()

internal/caches/list_file_hash_map_sqlite_test.go

@@ -4,6 +4,7 @@ package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/assert"
@@ -21,7 +22,7 @@ func TestFileListHashMap_Memory(t *testing.T) {
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
 
-	var m = caches.NewFileListHashMap()
+	var m = caches.NewSQLiteFileListHashMap()
 	m.SetIsAvailable(true)
 
 	for i := 0; i < 1_000_000; i++ {
@@ -86,7 +87,11 @@ func TestFileListHashMap_BigInt(t *testing.T) {
 }
 
 func TestFileListHashMap_Load(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
 
 	defer func() {
 		_ = list.Close()
@@ -97,7 +102,7 @@ func TestFileListHashMap_Load(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	var m = caches.NewFileListHashMap()
+	var m = caches.NewSQLiteFileListHashMap()
 	var before = time.Now()
 	var db = list.GetDB("abc")
 	err = m.Load(db)
@@ -116,7 +121,7 @@ func TestFileListHashMap_Load(t *testing.T) {
 func TestFileListHashMap_Delete(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	var m = caches.NewFileListHashMap()
+	var m = caches.NewSQLiteFileListHashMap()
 	m.SetIsReady(true)
 	m.SetIsAvailable(true)
 	m.Add("a")
@@ -126,7 +131,7 @@ func TestFileListHashMap_Delete(t *testing.T) {
 }
 
 func TestFileListHashMap_Clean(t *testing.T) {
-	var m = caches.NewFileListHashMap()
+	var m = caches.NewSQLiteFileListHashMap()
 	m.SetIsAvailable(true)
 	m.Clean()
 	m.Add("a")
@@ -144,7 +149,7 @@ func Benchmark_BigInt(b *testing.B) {
 }
 
 func BenchmarkFileListHashMap_Exist(b *testing.B) {
-	var m = caches.NewFileListHashMap()
+	var m = caches.NewSQLiteFileListHashMap()
 	m.SetIsAvailable(true)
 	m.SetIsReady(true)
 

internal/caches/list_file_kv.go

@@ -0,0 +1,311 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/goman"
+	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"testing"
+)
+
+const countKVStores = 10
+
+type KVFileList struct {
+	dir    string
+	stores [countKVStores]*KVListFileStore
+
+	onAdd    func(item *Item)
+	onRemove func(item *Item)
+}
+
+func NewKVFileList(dir string) *KVFileList {
+	dir = strings.TrimSuffix(dir, "/")
+
+	var stores = [countKVStores]*KVListFileStore{}
+	for i := 0; i < countKVStores; i++ {
+		stores[i] = NewKVListFileStore(dir + "/db-" + types.String(i) + ".store")
+	}
+
+	return &KVFileList{
+		dir:    dir,
+		stores: stores,
+	}
+}
+
+// Init 初始化
+func (this *KVFileList) Init() error {
+	remotelogs.Println("CACHE", "loading database from '"+this.dir+"' ...")
+
+	var group = goman.NewTaskGroup()
+	var lastErr error
+
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			err := storeCopy.Open()
+			if err != nil {
+				lastErr = fmt.Errorf("open store '"+storeCopy.Path()+"' failed: %w", err)
+			}
+		})
+	}
+	group.Wait()
+
+	return lastErr
+}
+
+// Reset 重置数据
+func (this *KVFileList) Reset() error {
+	// do nothing
+	return nil
+}
+
+// Add 添加内容
+func (this *KVFileList) Add(hash string, item *Item) error {
+	err := this.getStore(hash).AddItem(hash, item)
+	if err != nil {
+		return err
+	}
+
+	if this.onAdd != nil {
+		this.onAdd(item)
+	}
+
+	return nil
+}
+
+// Exist 检查内容是否存在
+func (this *KVFileList) Exist(hash string) (bool, error) {
+	return this.getStore(hash).ExistItem(hash)
+}
+
+// ExistQuick 快速检查内容是否存在
+func (this *KVFileList) ExistQuick(hash string) (bool, error) {
+	return this.getStore(hash).ExistQuickItem(hash)
+}
+
+// CleanPrefix 清除某个前缀的缓存
+func (this *KVFileList) CleanPrefix(prefix string) error {
+	var group = goman.NewTaskGroup()
+	var lastErr error
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			err := storeCopy.CleanItemsWithPrefix(prefix)
+			if err != nil {
+				lastErr = err
+			}
+		})
+	}
+	group.Wait()
+	return lastErr
+}
+
+// CleanMatchKey 清除通配符匹配的Key
+func (this *KVFileList) CleanMatchKey(key string) error {
+	var group = goman.NewTaskGroup()
+	var lastErr error
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			err := storeCopy.CleanItemsWithWildcardKey(key)
+			if err != nil {
+				lastErr = err
+			}
+		})
+	}
+	group.Wait()
+	return lastErr
+}
+
+// CleanMatchPrefix 清除通配符匹配的前缀
+func (this *KVFileList) CleanMatchPrefix(prefix string) error {
+	var group = goman.NewTaskGroup()
+	var lastErr error
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			err := storeCopy.CleanItemsWithWildcardPrefix(prefix)
+			if err != nil {
+				lastErr = err
+			}
+		})
+	}
+	group.Wait()
+	return lastErr
+}
+
+// Remove 删除内容
+func (this *KVFileList) Remove(hash string) error {
+	err := this.getStore(hash).RemoveItem(hash)
+	if err != nil {
+		return err
+	}
+
+	if this.onRemove != nil {
+		// when remove file item, no any extra information needed
+		this.onRemove(nil)
+	}
+
+	return nil
+}
+
+// Purge 清理过期数据
+func (this *KVFileList) Purge(count int, callback func(hash string) error) (int, error) {
+	count /= countKVStores
+	if count <= 0 {
+		count = 100
+	}
+
+	var countFound = 0
+	var lastErr error
+	for _, store := range this.stores {
+		purgeCount, err := store.PurgeItems(count, callback)
+		countFound += purgeCount
+		if err != nil {
+			lastErr = err
+		}
+	}
+
+	return countFound, lastErr
+}
+
+// PurgeLFU 清理LFU数据
+func (this *KVFileList) PurgeLFU(count int, callback func(hash string) error) error {
+	count /= countKVStores
+	if count <= 0 {
+		count = 100
+	}
+
+	var lastErr error
+	for _, store := range this.stores {
+		err := store.PurgeLFUItems(count, callback)
+		if err != nil {
+			lastErr = err
+		}
+	}
+	return lastErr
+}
+
+// CleanAll 清除所有缓存
+func (this *KVFileList) CleanAll() error {
+	var group = goman.NewTaskGroup()
+	var lastErr error
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			err := storeCopy.RemoveAllItems()
+			if err != nil {
+				lastErr = err
+			}
+		})
+	}
+	group.Wait()
+	return lastErr
+}
+
+// Stat 统计
+func (this *KVFileList) Stat(check func(hash string) bool) (*Stat, error) {
+	var stat = &Stat{}
+
+	var group = goman.NewTaskGroup()
+
+	var lastErr error
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			storeStat, err := storeCopy.StatItems()
+			if err != nil {
+				lastErr = err
+				return
+			}
+
+			group.Lock()
+			stat.Size += storeStat.Size
+			stat.ValueSize += storeStat.ValueSize
+			stat.Count += storeStat.Count
+			group.Unlock()
+		})
+	}
+
+	group.Wait()
+
+	return stat, lastErr
+}
+
+// Count 总数量
+func (this *KVFileList) Count() (int64, error) {
+	var count int64
+
+	var group = goman.NewTaskGroup()
+
+	var lastErr error
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			countStoreItems, err := storeCopy.CountItems()
+			if err != nil {
+				lastErr = err
+				return
+			}
+
+			group.Lock()
+			count += countStoreItems
+			group.Unlock()
+		})
+	}
+
+	group.Wait()
+
+	return count, lastErr
+}
+
+// OnAdd 添加事件
+func (this *KVFileList) OnAdd(fn func(item *Item)) {
+	this.onAdd = fn
+}
+
+// OnRemove 删除事件
+func (this *KVFileList) OnRemove(fn func(item *Item)) {
+	this.onRemove = fn
+}
+
+// Close 关闭
+func (this *KVFileList) Close() error {
+	var lastErr error
+	var group = goman.NewTaskGroup()
+	for _, store := range this.stores {
+		var storeCopy = store
+		group.Run(func() {
+			err := storeCopy.Close()
+			if err != nil {
+				lastErr = err
+			}
+		})
+	}
+	group.Wait()
+	return lastErr
+}
+
+// IncreaseHit 增加点击量
+func (this *KVFileList) IncreaseHit(hash string) error {
+	// do nothing
+	return nil
+}
+
+func (this *KVFileList) TestInspect(t *testing.T) error {
+	for _, store := range this.stores {
+		err := store.TestInspect(t)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (this *KVFileList) getStore(hash string) *KVListFileStore {
+	return this.stores[fnv.HashString(hash)%countKVStores]
+}

internal/caches/list_file_kv_objects.go

@@ -0,0 +1,66 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"strings"
+)
+
+// ItemKVEncoder item encoder
+type ItemKVEncoder[T interface{ *Item }] struct {
+}
+
+func NewItemKVEncoder[T interface{ *Item }]() *ItemKVEncoder[T] {
+	return &ItemKVEncoder[T]{}
+}
+
+func (this *ItemKVEncoder[T]) Encode(value T) ([]byte, error) {
+	return json.Marshal(value)
+}
+
+func (this *ItemKVEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
+	switch fieldName {
+	case "createdAt":
+		var b = make([]byte, 4)
+		var createdAt = any(value).(*Item).CreatedAt
+		binary.BigEndian.PutUint32(b, uint32(createdAt))
+		return b, nil
+	case "staleAt":
+		var b = make([]byte, 4)
+		var staleAt = any(value).(*Item).StaleAt
+		if staleAt < 0 {
+			staleAt = 0
+		}
+		binary.BigEndian.PutUint32(b, uint32(staleAt))
+		return b, nil
+	case "serverId":
+		var b = make([]byte, 4)
+		var serverId = any(value).(*Item).ServerId
+		if serverId < 0 {
+			serverId = 0
+		}
+		binary.BigEndian.PutUint32(b, uint32(serverId))
+		return b, nil
+	case "key":
+		return []byte(any(value).(*Item).Key), nil
+	case "wildKey":
+		var key = any(value).(*Item).Key
+		var dotIndex = strings.Index(key, ".")
+		if dotIndex > 0 {
+			var slashIndex = strings.LastIndex(key[:dotIndex], "/")
+			if slashIndex > 0 {
+				key = key[:dotIndex][:slashIndex+1] + "*" + key[dotIndex:]
+			}
+		}
+
+		return []byte(key), nil
+	}
+	return nil, nil
+}
+
+func (this *ItemKVEncoder[T]) Decode(valueBytes []byte) (value T, err error) {
+	err = json.Unmarshal(valueBytes, &value)
+	return
+}

internal/caches/list_file_kv_objects_test.go

@@ -0,0 +1,69 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/iwind/TeaGo/assert"
+	"testing"
+)
+
+func TestItemKVEncoder_EncodeField(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var encoder = caches.NewItemKVEncoder[*caches.Item]()
+	{
+		key, err := encoder.EncodeField(&caches.Item{
+			Key: "https://example.com/index.html",
+		}, "key")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("key:", string(key))
+		a.IsTrue(string(key) == "https://example.com/index.html")
+	}
+
+	{
+		key, err := encoder.EncodeField(&caches.Item{
+			Key: "",
+		}, "wildKey")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("key:", string(key))
+		a.IsTrue(string(key) == "")
+	}
+
+	{
+		key, err := encoder.EncodeField(&caches.Item{
+			Key: "example.com/index.html",
+		}, "wildKey")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("key:", string(key))
+		a.IsTrue(string(key) == "example.com/index.html")
+	}
+
+	{
+		key, err := encoder.EncodeField(&caches.Item{
+			Key: "https://example.com/index.html",
+		}, "wildKey")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("key:", string(key))
+		a.IsTrue(string(key) == "https://*.com/index.html")
+	}
+
+	{
+		key, err := encoder.EncodeField(&caches.Item{
+			Key: "https://www.example.com/index.html",
+		}, "wildKey")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("key:", string(key))
+		a.IsTrue(string(key) == "https://*.example.com/index.html")
+	}
+}

internal/caches/list_file_kv_store.go

@@ -0,0 +1,481 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches
+
+import (
+	"errors"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
+	"github.com/cockroachdb/pebble"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+type KVListFileStore struct {
+	path     string
+	rawStore *kvstore.Store
+
+	// tables
+	itemsTable *kvstore.Table[*Item]
+
+	rawIsReady bool
+}
+
+func NewKVListFileStore(path string) *KVListFileStore {
+	return &KVListFileStore{
+		path: path,
+	}
+}
+
+func (this *KVListFileStore) Open() error {
+	var reg = regexp.MustCompile(`^(.+)/([\w-]+)(\.store)$`)
+	var matches = reg.FindStringSubmatch(this.path)
+	if len(matches) != 4 {
+		return errors.New("invalid path '" + this.path + "'")
+	}
+	var dir = matches[1]
+	var name = matches[2]
+
+	rawStore, err := kvstore.OpenStoreDir(dir, name)
+	if err != nil {
+		return err
+	}
+	this.rawStore = rawStore
+
+	db, err := rawStore.NewDB("cache")
+	if err != nil {
+		return err
+	}
+
+	{
+		table, tableErr := kvstore.NewTable[*Item]("items", NewItemKVEncoder[*Item]())
+		if tableErr != nil {
+			return tableErr
+		}
+
+		err = table.AddFields("staleAt", "key", "wildKey", "createdAt")
+		if err != nil {
+			return err
+		}
+
+		db.AddTable(table)
+		this.itemsTable = table
+	}
+
+	this.rawIsReady = true
+
+	return nil
+}
+
+func (this *KVListFileStore) Path() string {
+	return this.path
+}
+
+func (this *KVListFileStore) AddItem(hash string, item *Item) error {
+	if !this.isReady() {
+		return nil
+	}
+
+	var currentTime = fasttime.Now().Unix()
+	if item.ExpiresAt <= currentTime {
+		return nil
+	}
+	if item.CreatedAt <= 0 {
+		item.CreatedAt = currentTime
+	}
+	if item.StaleAt <= 0 {
+		item.StaleAt = item.ExpiresAt + DefaultStaleCacheSeconds
+	}
+	return this.itemsTable.Set(hash, item)
+}
+
+func (this *KVListFileStore) ExistItem(hash string) (bool, error) {
+	if !this.isReady() {
+		return false, nil
+	}
+
+	item, err := this.itemsTable.Get(hash)
+	if err != nil {
+		if kvstore.IsKeyNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	if item == nil {
+		return false, nil
+	}
+
+	return item.ExpiresAt >= fasttime.NewFastTime().Unix(), nil
+}
+
+func (this *KVListFileStore) ExistQuickItem(hash string) (bool, error) {
+	if !this.isReady() {
+		return false, nil
+	}
+
+	return this.itemsTable.Exist(hash)
+}
+
+func (this *KVListFileStore) RemoveItem(hash string) error {
+	if !this.isReady() {
+		return nil
+	}
+
+	return this.itemsTable.Delete(hash)
+}
+
+func (this *KVListFileStore) RemoveAllItems() error {
+	if !this.isReady() {
+		return nil
+	}
+
+	return this.itemsTable.Truncate()
+}
+
+func (this *KVListFileStore) PurgeItems(count int, callback func(hash string) error) (int, error) {
+	if !this.isReady() {
+		return 0, nil
+	}
+
+	var countFound int
+	var currentTime = fasttime.Now().Unix()
+	var hashList []string
+	err := this.itemsTable.
+		Query().
+		FieldAsc("staleAt").
+		Limit(count).
+		FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
+			if item.Value == nil {
+				return true, nil
+			}
+			if item.Value.StaleAt < currentTime {
+				countFound++
+				hashList = append(hashList, item.Key)
+				return true, nil
+			}
+			return false, nil
+		})
+	if err != nil {
+		return 0, err
+	}
+
+	// delete items
+	if len(hashList) > 0 {
+		txErr := this.itemsTable.WriteTx(func(tx *kvstore.Tx[*Item]) error {
+			for _, hash := range hashList {
+				deleteErr := tx.Delete(hash)
+				if deleteErr != nil {
+					return deleteErr
+				}
+			}
+			return nil
+		})
+		if txErr != nil {
+			return 0, txErr
+		}
+
+		for _, hash := range hashList {
+			callbackErr := callback(hash)
+			if callbackErr != nil {
+				return 0, callbackErr
+			}
+		}
+	}
+
+	return countFound, nil
+}
+
+func (this *KVListFileStore) PurgeLFUItems(count int, callback func(hash string) error) error {
+	if !this.isReady() {
+		return nil
+	}
+
+	var hashList []string
+	err := this.itemsTable.
+		Query().
+		FieldAsc("createdAt").
+		Limit(count).
+		FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
+			if item.Value != nil {
+				hashList = append(hashList, item.Key)
+			}
+			return true, nil
+		})
+	if err != nil {
+		return err
+	}
+
+	// delete items
+	if len(hashList) > 0 {
+		txErr := this.itemsTable.WriteTx(func(tx *kvstore.Tx[*Item]) error {
+			for _, hash := range hashList {
+				deleteErr := tx.Delete(hash)
+				if deleteErr != nil {
+					return deleteErr
+				}
+			}
+			return nil
+		})
+		if txErr != nil {
+			return txErr
+		}
+
+		for _, hash := range hashList {
+			callbackErr := callback(hash)
+			if callbackErr != nil {
+				return callbackErr
+			}
+		}
+	}
+
+	return nil
+}
+
+func (this *KVListFileStore) CleanItemsWithPrefix(prefix string) error {
+	if !this.isReady() {
+		return nil
+	}
+
+	if len(prefix) == 0 {
+		return nil
+	}
+
+	var currentTime = fasttime.Now().Unix()
+
+	var fieldOffset []byte
+	const size = 1000
+	for {
+		var count int
+		err := this.itemsTable.
+			Query().
+			FieldPrefix("key", prefix).
+			FieldOffset(fieldOffset).
+			Limit(size).
+			ForUpdate().
+			FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
+				if item.Value == nil {
+					return true, nil
+				}
+
+				count++
+				fieldOffset = item.FieldKey
+
+				if item.Value.CreatedAt >= currentTime {
+					return true, nil
+				}
+				if item.Value.ExpiresAt == 0 {
+					return true, nil
+				}
+
+				item.Value.ExpiresAt = 0
+				item.Value.StaleAt = 0
+
+				setErr := tx.Set(item.Key, item.Value) // TODO improve performance
+				if setErr != nil {
+					return false, setErr
+				}
+
+				return true, nil
+			})
+		if err != nil {
+			return err
+		}
+
+		if count < size {
+			break
+		}
+	}
+
+	return nil
+}
+
+func (this *KVListFileStore) CleanItemsWithWildcardPrefix(prefix string) error {
+	if !this.isReady() {
+		return nil
+	}
+
+	if len(prefix) == 0 {
+		return nil
+	}
+
+	var currentTime = fasttime.Now().Unix()
+
+	var fieldOffset []byte
+	const size = 1000
+	for {
+		var count int
+		err := this.itemsTable.
+			Query().
+			FieldPrefix("wildKey", prefix).
+			FieldOffset(fieldOffset).
+			Limit(size).
+			ForUpdate().
+			FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
+				if item.Value == nil {
+					return true, nil
+				}
+
+				count++
+				fieldOffset = item.FieldKey
+
+				if item.Value.CreatedAt >= currentTime {
+					return true, nil
+				}
+				if item.Value.ExpiresAt == 0 {
+					return true, nil
+				}
+
+				item.Value.ExpiresAt = 0
+				item.Value.StaleAt = 0
+
+				setErr := tx.Set(item.Key, item.Value) // TODO improve performance
+				if setErr != nil {
+					return false, setErr
+				}
+
+				return true, nil
+			})
+		if err != nil {
+			return err
+		}
+
+		if count < size {
+			break
+		}
+	}
+
+	return nil
+}
+
+func (this *KVListFileStore) CleanItemsWithWildcardKey(key string) error {
+	if !this.isReady() {
+		return nil
+	}
+
+	if len(key) == 0 {
+		return nil
+	}
+
+	var currentTime = fasttime.Now().Unix()
+
+	for _, realKey := range []string{key, key + SuffixAll} {
+		var fieldOffset = append(this.itemsTable.FieldKey("wildKey"), '$')
+		fieldOffset = append(fieldOffset, realKey...)
+		const size = 1000
+
+		var wildKey string
+		if !strings.HasSuffix(realKey, SuffixAll) {
+			wildKey = string(append([]byte(realKey), 0, 0))
+		} else {
+			wildKey = realKey
+		}
+
+		for {
+			var count int
+			err := this.itemsTable.
+				Query().
+				FieldPrefix("wildKey", wildKey).
+				FieldOffset(fieldOffset).
+				Limit(size).
+				ForUpdate().
+				FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
+					if item.Value == nil {
+						return true, nil
+					}
+
+					count++
+					fieldOffset = item.FieldKey
+
+					if item.Value.CreatedAt >= currentTime {
+						return true, nil
+					}
+					if item.Value.ExpiresAt == 0 {
+						return true, nil
+					}
+
+					item.Value.ExpiresAt = 0
+					item.Value.StaleAt = 0
+
+					setErr := tx.Set(item.Key, item.Value) // TODO improve performance
+					if setErr != nil {
+						return false, setErr
+					}
+
+					return true, nil
+				})
+			if err != nil {
+				return err
+			}
+
+			if count < size {
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
+func (this *KVListFileStore) CountItems() (int64, error) {
+	if !this.isReady() {
+		return 0, nil
+	}
+
+	return this.itemsTable.Count()
+}
+
+func (this *KVListFileStore) StatItems() (*Stat, error) {
+	if !this.isReady() {
+		return &Stat{}, nil
+	}
+
+	var stat = &Stat{}
+
+	err := this.itemsTable.
+		Query().
+		FindAll(func(tx *kvstore.Tx[*Item], item kvstore.Item[*Item]) (goNext bool, err error) {
+			if item.Value != nil {
+				stat.Size += item.Value.Size()
+				stat.ValueSize += item.Value.BodySize
+				stat.Count++
+			}
+			return true, nil
+		})
+	return stat, err
+}
+
+func (this *KVListFileStore) TestInspect(t *testing.T) error {
+	if !this.isReady() {
+		return nil
+	}
+
+	it, err := this.rawStore.RawDB().NewIter(&pebble.IterOptions{})
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = it.Close()
+	}()
+
+	for it.First(); it.Valid(); it.Next() {
+		valueBytes, valueErr := it.ValueAndErr()
+		if valueErr != nil {
+			return valueErr
+		}
+		t.Log(string(it.Key()), "=>", string(valueBytes))
+	}
+	return nil
+}
+
+func (this *KVListFileStore) Close() error {
+	if this.rawStore != nil {
+		return this.rawStore.Close()
+	}
+
+	return nil
+}
+
+func (this *KVListFileStore) isReady() bool {
+	return this.rawIsReady && !this.rawStore.IsClosed()
+}

internal/caches/list_file_kv_test.go

@@ -0,0 +1,331 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/iwind/TeaGo/Tea"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	stringutil "github.com/iwind/TeaGo/utils/string"
+	"math/rand"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+)
+
+var testingKVList *caches.KVFileList
+
+func TestMain(m *testing.M) {
+	m.Run()
+
+	if testingKVList != nil {
+		_ = testingKVList.Close()
+	}
+}
+
+func testOpenKVFileList(t *testing.T) *caches.KVFileList {
+	var list = caches.NewKVFileList(Tea.Root + "/data/stores/cache-stores")
+	err := list.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testingKVList = list
+	return list
+}
+
+func TestNewKVFileList(t *testing.T) {
+	_ = testOpenKVFileList(t)
+}
+
+func TestKVFileList_Add(t *testing.T) {
+	var list = testOpenKVFileList(t)
+
+	err := list.Add(stringutil.Md5("123456"), &caches.Item{
+		Type:       caches.ItemTypeFile,
+		Key:        "https://example.com/index.html",
+		ExpiresAt:  time.Now().Unix() + 60,
+		StaleAt:    0,
+		HeaderSize: 0,
+		BodySize:   4096,
+		MetaSize:   0,
+		Host:       "",
+		ServerId:   1,
+		Week:       0,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestKVFileList_Add_Many(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = testOpenKVFileList(t)
+
+	const start = 0
+	const count = 1_000_000
+	const concurrent = 100
+
+	var before = time.Now()
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", fmt.Sprintf("%.2fs", costSeconds), "qps:", fmt.Sprintf("%.2fK/s", float64(count)/1000/costSeconds))
+	}()
+
+	var wg = &sync.WaitGroup{}
+	wg.Add(concurrent)
+	for c := 0; c < concurrent; c++ {
+		go func(c int) {
+			defer wg.Done()
+
+			var segmentStart = start + count/concurrent*c
+			for i := segmentStart; i < segmentStart+count/concurrent; i++ {
+				err := list.Add(stringutil.Md5(strconv.Itoa(i)), &caches.Item{
+					Type:       caches.ItemTypeFile,
+					Key:        "https://www.example.com/index.html" + strconv.Itoa(i),
+					ExpiresAt:  time.Now().Unix() + 60,
+					StaleAt:    0,
+					HeaderSize: 0,
+					BodySize:   int64(rand.Int() % 1_000_000),
+					MetaSize:   0,
+					Host:       "",
+					ServerId:   1,
+					Week:       0,
+				})
+				if err != nil {
+					t.Log(err)
+				}
+			}
+		}(c)
+	}
+	wg.Wait()
+}
+
+func TestKVFileList_Add_Many_Suffix(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = testOpenKVFileList(t)
+
+	const start = 0
+	const count = 1000
+	const concurrent = 100
+
+	var before = time.Now()
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", fmt.Sprintf("%.2fs", costSeconds), "qps:", fmt.Sprintf("%.2fK/s", float64(count)/1000/costSeconds))
+	}()
+
+	var wg = &sync.WaitGroup{}
+	wg.Add(concurrent)
+	for c := 0; c < concurrent; c++ {
+		go func(c int) {
+			defer wg.Done()
+
+			var segmentStart = start + count/concurrent*c
+			for i := segmentStart; i < segmentStart+count/concurrent; i++ {
+				err := list.Add(stringutil.Md5(strconv.Itoa(i)+caches.SuffixAll), &caches.Item{
+					Type:       caches.ItemTypeFile,
+					Key:        "https://www.example.com/index.html" + strconv.Itoa(i) + caches.SuffixAll + "zip",
+					ExpiresAt:  time.Now().Unix() + 60,
+					StaleAt:    0,
+					HeaderSize: 0,
+					BodySize:   int64(rand.Int() % 1_000_000),
+					MetaSize:   0,
+					Host:       "",
+					ServerId:   1,
+					Week:       0,
+				})
+				if err != nil {
+					t.Log(err)
+				}
+			}
+		}(c)
+	}
+	wg.Wait()
+}
+
+func TestKVFileList_Exist(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	for _, hash := range []string{
+		stringutil.Md5("123456"),
+		stringutil.Md5("654321"),
+	} {
+		b, err := list.Exist(hash)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(hash, "=>", b)
+	}
+}
+
+func TestKVFileList_ExistQuick(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	for _, hash := range []string{
+		stringutil.Md5("123456"),
+		stringutil.Md5("654321"),
+	} {
+		b, err := list.ExistQuick(hash)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(hash, "=>", b)
+	}
+}
+
+func TestKVFileList_Remove(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	for _, hash := range []string{
+		stringutil.Md5("123456"),
+		stringutil.Md5("654321"),
+	} {
+		err := list.Remove(hash)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestKVFileList_CleanAll(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	err := list.CleanAll()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestKVFileList_Inspect(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = testOpenKVFileList(t)
+	err := list.TestInspect(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestKVFileList_Purge(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	var before = time.Now()
+	count, err := list.Purge(4_000, func(hash string) error {
+		//t.Log("hash:", hash)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "count:", count)
+}
+
+func TestKVFileList_PurgeLFU(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	var before = time.Now()
+	err := list.PurgeLFU(20000, func(hash string) error {
+		t.Log("hash:", hash)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000))
+}
+
+func TestKVFileList_Count(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	var before = time.Now()
+	count, err := list.Count()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "count:", count)
+}
+
+func TestKVFileList_Stat(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	var before = time.Now()
+	stat, err := list.Stat(func(hash string) bool {
+		return true
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "stat:", fmt.Sprintf("%+v", stat))
+}
+
+func TestKVFileList_CleanPrefix(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	var before = time.Now()
+
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000))
+	}()
+
+	err := list.CleanPrefix("https://www.example.com/index.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestKVFileList_CleanMatchPrefix(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	var before = time.Now()
+
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000))
+	}()
+
+	err := list.CleanMatchPrefix("https://*.example.com/index.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestKVFileList_CleanMatchKey(t *testing.T) {
+	var list = testOpenKVFileList(t)
+	var before = time.Now()
+
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000))
+	}()
+
+	err := list.CleanMatchKey("https://*.example.com/index.html123")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func BenchmarkKVFileList_Exist(b *testing.B) {
+	var list = caches.NewKVFileList(Tea.Root + "/data/stores/cache-stores")
+	err := list.Init()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	defer func() {
+		_ = list.Close()
+	}()
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, existErr := list.Exist(stringutil.Md5(strconv.Itoa(rand.Int() % 2_000_000)))
+			if existErr != nil {
+				b.Fatal(existErr)
+			}
+		}
+	})
+}

internal/caches/list_file_sqlite.go

@@ -21,10 +21,10 @@ import (
 
 const CountFileDB = 20
 
-// FileList 文件缓存列表管理
-type FileList struct {
+// SQLiteFileList 文件缓存列表管理
+type SQLiteFileList struct {
 	dir    string
-	dbList [CountFileDB]*FileListDB
+	dbList [CountFileDB]*SQLiteFileListDB
 
 	onAdd    func(item *Item)
 	onRemove func(item *Item)
@@ -35,18 +35,18 @@ type FileList struct {
 	oldDir string
 }
 
-func NewFileList(dir string) ListInterface {
-	return &FileList{
+func NewSQLiteFileList(dir string) ListInterface {
+	return &SQLiteFileList{
 		dir:         dir,
 		memoryCache: ttlcache.NewCache[zero.Zero](),
 	}
 }
 
-func (this *FileList) SetOldDir(oldDir string) {
+func (this *SQLiteFileList) SetOldDir(oldDir string) {
 	this.oldDir = oldDir
 }
 
-func (this *FileList) Init() error {
+func (this *SQLiteFileList) Init() error {
 	// 检查目录是否存在
 	_, err := os.Stat(this.dir)
 	if err != nil {
@@ -73,7 +73,7 @@ func (this *FileList) Init() error {
 		go func(i int) {
 			defer wg.Done()
 
-			var db = NewFileListDB()
+			var db = NewSQLiteFileListDB()
 			dbErr := db.Open(dir + "/db-" + types.String(i) + ".db")
 			if dbErr != nil {
 				lastErr = dbErr
@@ -105,12 +105,12 @@ func (this *FileList) Init() error {
 	return nil
 }
 
-func (this *FileList) Reset() error {
+func (this *SQLiteFileList) Reset() error {
 	// 不做任何事情
 	return nil
 }
 
-func (this *FileList) Add(hash string, item *Item) error {
+func (this *SQLiteFileList) Add(hash string, item *Item) error {
 	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
@@ -122,7 +122,7 @@ func (this *FileList) Add(hash string, item *Item) error {
 		return err
 	}
 
-	this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiredAt))
+	this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiresAt))
 
 	if this.onAdd != nil {
 		this.onAdd(item)
@@ -130,7 +130,7 @@ func (this *FileList) Add(hash string, item *Item) error {
 	return nil
 }
 
-func (this *FileList) Exist(hash string) (bool, error) {
+func (this *SQLiteFileList) Exist(hash string) (bool, error) {
 	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
@@ -160,11 +160,16 @@ func (this *FileList) Exist(hash string) (bool, error) {
 		}
 		return false, err
 	}
+
+	if expiredAt < fasttime.Now().Unix() {
+		return false, nil
+	}
+
 	this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(expiredAt))
 	return true, nil
 }
 
-func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
+func (this *SQLiteFileList) ExistQuick(hash string) (isReady bool, found bool) {
 	var db = this.GetDB(hash)
 
 	if !db.IsReady() || !db.HashMapIsLoaded() {
@@ -177,7 +182,7 @@ func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
 }
 
 // CleanPrefix 清理某个前缀的缓存数据
-func (this *FileList) CleanPrefix(prefix string) error {
+func (this *SQLiteFileList) CleanPrefix(prefix string) error {
 	if len(prefix) == 0 {
 		return nil
 	}
@@ -197,7 +202,7 @@ func (this *FileList) CleanPrefix(prefix string) error {
 }
 
 // CleanMatchKey 清理通配符匹配的缓存数据，类似于 https://*.example.com/hello
-func (this *FileList) CleanMatchKey(key string) error {
+func (this *SQLiteFileList) CleanMatchKey(key string) error {
 	if len(key) == 0 {
 		return nil
 	}
@@ -217,7 +222,7 @@ func (this *FileList) CleanMatchKey(key string) error {
 }
 
 // CleanMatchPrefix 清理通配符匹配的缓存数据，类似于 https://*.example.com/prefix/
-func (this *FileList) CleanMatchPrefix(prefix string) error {
+func (this *SQLiteFileList) CleanMatchPrefix(prefix string) error {
 	if len(prefix) == 0 {
 		return nil
 	}
@@ -236,7 +241,7 @@ func (this *FileList) CleanMatchPrefix(prefix string) error {
 	return nil
 }
 
-func (this *FileList) Remove(hash string) error {
+func (this *SQLiteFileList) Remove(hash string) error {
 	_, err := this.remove(hash, false)
 	return err
 }
@@ -244,7 +249,7 @@ func (this *FileList) Remove(hash string) error {
 // Purge 清理过期的缓存
 // count 每次遍历的最大数量，控制此数字可以保证每次清理的时候不用花太多时间
 // callback 每次发现过期key的调用
-func (this *FileList) Purge(count int, callback func(hash string) error) (int, error) {
+func (this *SQLiteFileList) Purge(count int, callback func(hash string) error) (int, error) {
 	count /= CountFileDB
 	if count <= 0 {
 		count = 100
@@ -285,7 +290,7 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
 	return countFound, nil
 }
 
-func (this *FileList) PurgeLFU(count int, callback func(hash string) error) error {
+func (this *SQLiteFileList) PurgeLFU(count int, callback func(hash string) error) error {
 	count /= CountFileDB
 	if count <= 0 {
 		count = 100
@@ -322,7 +327,7 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
 	return nil
 }
 
-func (this *FileList) CleanAll() error {
+func (this *SQLiteFileList) CleanAll() error {
 	defer this.memoryCache.Clean()
 
 	for _, db := range this.dbList {
@@ -335,7 +340,7 @@ func (this *FileList) CleanAll() error {
 	return nil
 }
 
-func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
+func (this *SQLiteFileList) Stat(check func(hash string) bool) (*Stat, error) {
 	var result = &Stat{}
 
 	for _, db := range this.dbList {
@@ -365,7 +370,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
 
 // Count 总数量
 // 常用的方法，所以避免直接查询数据库
-func (this *FileList) Count() (int64, error) {
+func (this *SQLiteFileList) Count() (int64, error) {
 	var total int64
 	for _, db := range this.dbList {
 		count, err := db.Total()
@@ -378,7 +383,7 @@ func (this *FileList) Count() (int64, error) {
 }
 
 // IncreaseHit 增加点击量
-func (this *FileList) IncreaseHit(hash string) error {
+func (this *SQLiteFileList) IncreaseHit(hash string) error {
 	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
@@ -389,36 +394,41 @@ func (this *FileList) IncreaseHit(hash string) error {
 }
 
 // OnAdd 添加事件
-func (this *FileList) OnAdd(f func(item *Item)) {
+func (this *SQLiteFileList) OnAdd(f func(item *Item)) {
 	this.onAdd = f
 }
 
 // OnRemove 删除事件
-func (this *FileList) OnRemove(f func(item *Item)) {
+func (this *SQLiteFileList) OnRemove(f func(item *Item)) {
 	this.onRemove = f
 }
 
-func (this *FileList) Close() error {
+func (this *SQLiteFileList) Close() error {
 	this.memoryCache.Destroy()
 
+	var group = goman.NewTaskGroup()
 	for _, db := range this.dbList {
-		if db != nil {
-			_ = db.Close()
-		}
+		var dbCopy = db
+		group.Run(func() {
+			if dbCopy != nil {
+				_ = dbCopy.Close()
+			}
+		})
 	}
+	group.Wait()
 
 	return nil
 }
 
-func (this *FileList) GetDBIndex(hash string) uint64 {
+func (this *SQLiteFileList) GetDBIndex(hash string) uint64 {
 	return fnv.HashString(hash) % CountFileDB
 }
 
-func (this *FileList) GetDB(hash string) *FileListDB {
+func (this *SQLiteFileList) GetDB(hash string) *SQLiteFileListDB {
 	return this.dbList[fnv.HashString(hash)%CountFileDB]
 }
 
-func (this *FileList) HashMapIsLoaded() bool {
+func (this *SQLiteFileList) HashMapIsLoaded() bool {
 	for _, db := range this.dbList {
 		if !db.HashMapIsLoaded() {
 			return false
@@ -427,7 +437,7 @@ func (this *FileList) HashMapIsLoaded() bool {
 	return true
 }
 
-func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
+func (this *SQLiteFileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
 	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
@@ -459,14 +469,14 @@ func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err er
 }
 
 // 升级老版本数据库
-func (this *FileList) upgradeOldDB() {
+func (this *SQLiteFileList) upgradeOldDB() {
 	if len(this.oldDir) == 0 {
 		return
 	}
 	_ = this.UpgradeV3(this.oldDir, false)
 }
 
-func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
+func (this *SQLiteFileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 	// index.db
 	var indexDBPath = oldDir + "/index.db"
 	_, err := os.Stat(indexDBPath)
@@ -538,7 +548,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 				err = this.Add(hash, &Item{
 					Type:       ItemTypeFile,
 					Key:        key,
-					ExpiredAt:  expiredAt,
+					ExpiresAt:  expiredAt,
 					StaleAt:    staleAt,
 					HeaderSize: headerSize,
 					BodySize:   bodySize,
@@ -565,7 +575,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 	return nil
 }
 
-func (this *FileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
+func (this *SQLiteFileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
 	var maxTimestamp = fasttime.Now().Unix() + 3600
 	if expiresAt > maxTimestamp {
 		return maxTimestamp

internal/caches/list_file_sqlite_test.go

@@ -17,7 +17,11 @@ import (
 )
 
 func TestFileList_Init(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
 		_ = list.Close()
@@ -34,7 +38,11 @@ func TestFileList_Init(t *testing.T) {
 }
 
 func TestFileList_Add(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
 
 	defer func() {
 		_ = list.Close()
@@ -53,7 +61,7 @@ func TestFileList_Add(t *testing.T) {
 	t.Log("db index:", list.GetDBIndex(hash))
 	err = list.Add(hash, &caches.Item{
 		Key:        "123456",
-		ExpiredAt:  time.Now().Unix() + 1,
+		ExpiresAt:  time.Now().Unix() + 1,
 		HeaderSize: 1,
 		MetaSize:   2,
 		BodySize:   3,
@@ -74,7 +82,7 @@ func TestFileList_Add_Many(t *testing.T) {
 		return
 	}
 
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
 		_ = list.Close()
@@ -86,11 +94,13 @@ func TestFileList_Add_Many(t *testing.T) {
 	}
 
 	var before = time.Now()
-	for i := 0; i < 10_000_000; i++ {
+	const offset = 0
+	const count = 1_000_000
+	for i := offset; i < offset+count; i++ {
 		u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
 		_ = list.Add(stringutil.Md5(u), &caches.Item{
 			Key:        u,
-			ExpiredAt:  time.Now().Unix() + 3600,
+			ExpiresAt:  time.Now().Unix() + 3600,
 			HeaderSize: 1,
 			MetaSize:   2,
 			BodySize:   3,
@@ -107,7 +117,11 @@ func TestFileList_Add_Many(t *testing.T) {
 }
 
 func TestFileList_Exist(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
 	defer func() {
 		_ = list.Close()
 	}()
@@ -143,10 +157,14 @@ func TestFileList_Exist(t *testing.T) {
 }
 
 func TestFileList_Exist_Many_DB(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	// 测试在多个数据库下的性能
 	var listSlice = []caches.ListInterface{}
 	for i := 1; i <= 10; i++ {
-		var list = caches.NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
+		var list = caches.NewSQLiteFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
 		err := list.Init()
 		if err != nil {
 			t.Fatal(err)
@@ -202,7 +220,11 @@ func TestFileList_Exist_Many_DB(t *testing.T) {
 }
 
 func TestFileList_CleanPrefix(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
 		_ = list.Close()
@@ -222,7 +244,11 @@ func TestFileList_CleanPrefix(t *testing.T) {
 }
 
 func TestFileList_Remove(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
 	defer func() {
 		_ = list.Close()
 	}()
@@ -246,7 +272,11 @@ func TestFileList_Remove(t *testing.T) {
 }
 
 func TestFileList_Purge(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
 		_ = list.Close()
@@ -270,7 +300,11 @@ func TestFileList_Purge(t *testing.T) {
 }
 
 func TestFileList_PurgeLFU(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
 		_ = list.Close()
@@ -294,7 +328,11 @@ func TestFileList_PurgeLFU(t *testing.T) {
 }
 
 func TestFileList_Stat(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
 		_ = list.Close()
@@ -313,7 +351,11 @@ func TestFileList_Stat(t *testing.T) {
 }
 
 func TestFileList_Count(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data")
 
 	defer func() {
 		_ = list.Close()
@@ -333,7 +375,11 @@ func TestFileList_Count(t *testing.T) {
 }
 
 func TestFileList_CleanAll(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data")
 
 	defer func() {
 		_ = list.Close()
@@ -352,7 +398,11 @@ func TestFileList_CleanAll(t *testing.T) {
 }
 
 func TestFileList_UpgradeV3(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p43").(*caches.FileList)
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p43").(*caches.SQLiteFileList)
 
 	defer func() {
 		_ = list.Close()
@@ -376,7 +426,11 @@ func TestFileList_UpgradeV3(t *testing.T) {
 }
 
 func BenchmarkFileList_Exist(b *testing.B) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
 		_ = list.Close()

internal/caches/list_memory.go

@@ -115,7 +115,7 @@ func (this *MemoryList) CleanPrefix(prefix string) error {
 	for _, itemMap := range this.itemMaps {
 		for _, item := range itemMap {
 			if strings.HasPrefix(item.Key, prefix) {
-				item.ExpiredAt = 0
+				item.ExpiresAt = 0
 			}
 		}
 	}
@@ -153,7 +153,7 @@ func (this *MemoryList) CleanMatchKey(key string) error {
 			if configutils.MatchDomain(host, item.Host) {
 				var itemRequestURI = item.RequestURI()
 				if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) {
-					item.ExpiredAt = 0
+					item.ExpiresAt = 0
 				}
 			}
 		}
@@ -189,7 +189,7 @@ func (this *MemoryList) CleanMatchPrefix(prefix string) error {
 			if configutils.MatchDomain(host, item.Host) {
 				var itemRequestURI = item.RequestURI()
 				if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) {
-					item.ExpiredAt = 0
+					item.ExpiresAt = 0
 				}
 			}
 		}
@@ -400,7 +400,19 @@ func (this *MemoryList) IncreaseHit(hash string) error {
 	return nil
 }
 
-func (this *MemoryList) print(t *testing.T) {
+func (this *MemoryList) Prefixes() []string {
+	return this.prefixes
+}
+
+func (this *MemoryList) ItemMaps() map[string]map[string]*Item {
+	return this.itemMaps
+}
+
+func (this *MemoryList) PurgeIndex() int {
+	return this.purgeIndex
+}
+
+func (this *MemoryList) Print(t *testing.T) {
 	this.locker.Lock()
 	for _, itemMap := range this.itemMaps {
 		if len(itemMap) > 0 {

internal/caches/list_memory_test.go

@@ -1,12 +1,14 @@
-package caches
+package caches_test
 
 import (
 	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/cespare/xxhash"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
+	stringutil "github.com/iwind/TeaGo/utils/string"
 	"math/rand"
 	"sort"
 	"strconv"
@@ -15,94 +17,99 @@ import (
 )
 
 func TestMemoryList_Add(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("123456", &Item{
+	_ = list.Add("123456", &caches.Item{
 		Key:        "c1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	t.Log(list.prefixes)
-	logs.PrintAsJSON(list.itemMaps, t)
+	t.Log(list.Prefixes())
+	logs.PrintAsJSON(list.ItemMaps(), t)
 	t.Log(list.Count())
 }
 
 func TestMemoryList_Remove(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
 	_ = list.Remove("b")
-	list.print(t)
+	list.Print(t)
 	t.Log(list.Count())
 }
 
 func TestMemoryList_Purge(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("c", &Item{
+	_ = list.Add("c", &caches.Item{
 		Key:        "c1",
-		ExpiredAt:  time.Now().Unix() - 3600,
+		ExpiresAt:  time.Now().Unix() - 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("d", &Item{
+	_ = list.Add("d", &caches.Item{
 		Key:        "d1",
-		ExpiredAt:  time.Now().Unix() - 2,
+		ExpiresAt:  time.Now().Unix() - 2,
 		HeaderSize: 1024,
 	})
 	_, _ = list.Purge(100, func(hash string) error {
 		t.Log("delete:", hash)
 		return nil
 	})
-	list.print(t)
+	list.Print(t)
 
 	for i := 0; i < 1000; i++ {
 		_, _ = list.Purge(100, func(hash string) error {
 			t.Log("delete:", hash)
 			return nil
 		})
-		t.Log(list.purgeIndex)
+		t.Log(list.PurgeIndex())
 	}
 
 	t.Log(list.Count())
 }
 
 func TestMemoryList_Purge_Large_List(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	var list = caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
 
-	for i := 0; i < 1_000_000; i++ {
-		_ = list.Add("a"+strconv.Itoa(i), &Item{
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 1_000_000
+	}
+
+	for i := 0; i < count; i++ {
+		_ = list.Add("a"+strconv.Itoa(i), &caches.Item{
 			Key:        "a" + strconv.Itoa(i),
-			ExpiredAt:  time.Now().Unix() + int64(rands.Int(0, 24*3600)),
+			ExpiresAt:  time.Now().Unix() + int64(rands.Int(0, 24*3600)),
 			HeaderSize: 1024,
 		})
 	}
@@ -113,45 +120,48 @@ func TestMemoryList_Purge_Large_List(t *testing.T) {
 }
 
 func TestMemoryList_Stat(t *testing.T) {
-	list := NewMemoryList()
+	list := caches.NewMemoryList()
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
-		ExpiredAt:  time.Now().Unix() + 3600,
+		ExpiresAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("c", &Item{
+	_ = list.Add("c", &caches.Item{
 		Key:        "c1",
-		ExpiredAt:  time.Now().Unix(),
+		ExpiresAt:  time.Now().Unix(),
 		HeaderSize: 1024,
 	})
-	_ = list.Add("d", &Item{
+	_ = list.Add("d", &caches.Item{
 		Key:        "d1",
-		ExpiredAt:  time.Now().Unix() - 2,
+		ExpiresAt:  time.Now().Unix() - 2,
 		HeaderSize: 1024,
 	})
 	result, _ := list.Stat(func(hash string) bool {
 		// 随机测试
-		rand.Seed(time.Now().UnixNano())
 		return rand.Int()%2 == 0
 	})
 	t.Log(result)
 }
 
 func TestMemoryList_CleanPrefix(t *testing.T) {
-	list := NewMemoryList()
+	list := caches.NewMemoryList()
 	_ = list.Init()
 	before := time.Now()
-	for i := 0; i < 1_000_000; i++ {
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 1_000_000
+	}
+	for i := 0; i < count; i++ {
 		key := "https://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
-		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
+		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
 			Key:        key,
-			ExpiredAt:  time.Now().Unix() + 3600,
+			ExpiresAt:  time.Now().Unix() + 3600,
 			BodySize:   0,
 			HeaderSize: 0,
 		})
@@ -174,7 +184,12 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
 func TestMapRandomDelete(t *testing.T) {
 	var countMap = map[int]int{} // k => count
 
-	for j := 0; j < 1_000_000; j++ {
+	var count = 1000
+	if testutils.IsSingleTesting() {
+		count = 1_000_000
+	}
+
+	for j := 0; j < count; j++ {
 		var m = map[int]bool{}
 		for i := 0; i < 100; i++ {
 			m[i] = true
@@ -203,18 +218,18 @@ func TestMapRandomDelete(t *testing.T) {
 }
 
 func TestMemoryList_PurgeLFU(t *testing.T) {
-	var list = NewMemoryList().(*MemoryList)
+	var list = caches.NewMemoryList().(*caches.MemoryList)
 
 	var before = time.Now()
 	defer func() {
 		t.Log(time.Since(before).Seconds()*1000, "ms")
 	}()
 
-	_ = list.Add("1", &Item{})
-	_ = list.Add("2", &Item{})
-	_ = list.Add("3", &Item{})
-	_ = list.Add("4", &Item{})
-	_ = list.Add("5", &Item{})
+	_ = list.Add("1", &caches.Item{})
+	_ = list.Add("2", &caches.Item{})
+	_ = list.Add("3", &caches.Item{})
+	_ = list.Add("4", &caches.Item{})
+	_ = list.Add("5", &caches.Item{})
 
 	//_ = list.IncreaseHit("1")
 	//_ = list.IncreaseHit("2")
@@ -245,29 +260,32 @@ func TestMemoryList_PurgeLFU(t *testing.T) {
 }
 
 func TestMemoryList_CleanAll(t *testing.T) {
-	var list = NewMemoryList().(*MemoryList)
-	_ = list.Add("a", &Item{})
+	var list = caches.NewMemoryList().(*caches.MemoryList)
+	_ = list.Add("a", &caches.Item{})
 	_ = list.CleanAll()
-	logs.PrintAsJSON(list.itemMaps, t)
+	logs.PrintAsJSON(list.ItemMaps(), t)
 	t.Log(list.Count())
 }
 
 func TestMemoryList_GC(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
 	for i := 0; i < 1_000_000; i++ {
 		key := "https://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
-		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
+		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
 			Key:        key,
-			ExpiredAt:  0,
+			ExpiresAt:  0,
 			BodySize:   0,
 			HeaderSize: 0,
 		})
 	}
-	time.Sleep(10 * time.Second)
-	t.Log("clean...", len(list.itemMaps))
+	t.Log("clean...", len(list.ItemMaps()))
 	_ = list.CleanAll()
-	t.Log("cleanAll...", len(list.itemMaps))
+	t.Log("cleanAll...", len(list.ItemMaps()))
 	before := time.Now()
 	//runtime.GC()
 	t.Log("gc cost:", time.Since(before).Seconds()*1000, "ms")
@@ -280,3 +298,30 @@ func TestMemoryList_GC(t *testing.T) {
 		time.Sleep(30 * time.Minute)
 	}
 }
+
+func BenchmarkMemoryList(b *testing.B) {
+	var list = caches.NewMemoryList()
+	err := list.Init()
+	if err != nil {
+		b.Fatal(err)
+	}
+	for i := 0; i < 1_000_000; i++ {
+		_ = list.Add(stringutil.Md5(types.String(i)), &caches.Item{
+			Key:        "a1",
+			ExpiresAt:  time.Now().Unix() + 3600,
+			HeaderSize: 1024,
+		})
+	}
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, _ = list.Exist(types.String("a" + types.String(rands.Int(1, 10000))))
+			_ = list.Add("a"+types.String(rands.Int(1, 100000)), &caches.Item{})
+			_, _ = list.Purge(1000, func(hash string) error {
+				return nil
+			})
+		}
+	})
+}

internal/caches/storage_file.go

@@ -111,6 +111,15 @@ func (this *FileStorage) Policy() *serverconfigs.HTTPCachePolicy {
 
 // CanUpdatePolicy 检查策略是否可以更新
 func (this *FileStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) bool {
+	if newPolicy == nil {
+		return false
+	}
+
+	// 检查类型
+	if newPolicy.Type != serverconfigs.CachePolicyStorageFile {
+		return false
+	}
+
 	// 检查路径是否有变化
 	oldOptionsJSON, err := json.Marshal(this.policy.Options)
 	if err != nil {
@@ -249,12 +258,25 @@ func (this *FileStorage) Init() error {
 		return errors.New("[CACHE]cache storage dir can not be empty")
 	}
 
-	var list = NewFileList(dir + "/p" + types.String(this.policy.Id) + "/.indexes")
-	err = list.Init()
-	if err != nil {
-		return err
+	// read list
+	var list ListInterface
+	var sqliteIndexesDir = dir + "/p" + types.String(this.policy.Id) + "/.indexes"
+	_, sqliteIndexesDirErr := os.Stat(sqliteIndexesDir)
+	if sqliteIndexesDirErr == nil || !teaconst.EnableKVCacheStore {
+		list = NewSQLiteFileList(sqliteIndexesDir)
+		err = list.Init()
+		if err != nil {
+			return err
+		}
+		list.(*SQLiteFileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
+	} else {
+		list = NewKVFileList(dir + "/p" + types.String(this.policy.Id) + "/.stores")
+		err = list.Init()
+		if err != nil {
+			return err
+		}
 	}
-	list.(*FileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
+
 	this.list = list
 
 	// 检查目录是否存在
@@ -455,7 +477,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	_, ok := sharedWritingFileKeyMap[key]
 	if ok {
 		sharedWritingFileKeyLocker.Unlock()
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w(001)", ErrFileIsWriting)
 	}
 
 	if !isFlushing && !fsutils.WriteReady() {
@@ -496,7 +518,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	// 检查两次写入缓存的时间是否过于相近，分片内容不受此限制
 	if err == nil && !isPartial && time.Since(stat.ModTime()) <= 1*time.Second {
 		// 防止并发连续写入
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w(002)", ErrFileIsWriting)
 	}
 
 	// 构造文件名
@@ -597,7 +619,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	err = syscall.Flock(int(writer.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
 	if err != nil {
 		removeOnFailure = false
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w (003)", ErrFileIsWriting)
 	}
 
 	var metaBodySize int64 = -1
@@ -1285,7 +1307,7 @@ func (this *FileStorage) hotLoop() {
 				Type:       writer.ItemType(),
 				Key:        item.Key,
 				Host:       ParseHost(item.Key),
-				ExpiredAt:  expiresAt,
+				ExpiresAt:  expiresAt,
 				HeaderSize: writer.HeaderSize(),
 				BodySize:   writer.BodySize(),
 			})
@@ -1599,7 +1621,8 @@ func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
 // ScanGarbageCaches 清理目录中“失联”的缓存文件
 // “失联”为不在HashMap中的文件
 func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error) error {
-	if !this.list.(*FileList).HashMapIsLoaded() {
+	_, isSQLite := this.list.(*SQLiteFileList)
+	if isSQLite && !this.list.(*SQLiteFileList).HashMapIsLoaded() {
 		return errors.New("cache list is loading")
 	}
 
@@ -1632,8 +1655,8 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
 				continue
 			}
 
-			dir2Matches, err := filepath.Glob(dir1 + "/*")
-			if err != nil {
+			dir2Matches, globErr := filepath.Glob(dir1 + "/*")
+			if globErr != nil {
 				// ignore error
 				continue
 			}
@@ -1656,8 +1679,8 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
 					}
 				}
 
-				fileMatches, err := filepath.Glob(dir2 + "/*.cache")
-				if err != nil {
+				fileMatches, globDir2Err := filepath.Glob(dir2 + "/*.cache")
+				if globDir2Err != nil {
 					// ignore error
 					continue
 				}
@@ -1669,7 +1692,19 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
 						continue
 					}
 
-					isReady, found := this.list.(*FileList).ExistQuick(hash)
+					var isReady, found bool
+					switch rawList := this.list.(type) {
+					case *SQLiteFileList:
+						isReady, found = rawList.ExistQuick(hash)
+					case *KVFileList:
+						isReady = true
+						var checkErr error
+						found, checkErr = rawList.ExistQuick(hash)
+						if checkErr != nil {
+							return checkErr
+						}
+					}
+
 					if !isReady {
 						continue
 					}
@@ -1679,8 +1714,8 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
 					}
 
 					// 检查文件正在被写入
-					stat, err := os.Stat(file)
-					if err != nil {
+					stat, statErr := os.Stat(file)
+					if statErr != nil {
 						continue
 					}
 					if fasttime.Now().Unix()-stat.ModTime().Unix() < 300 /** 5 minutes **/ {
@@ -1689,9 +1724,9 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
 
 					if fileCallback != nil {
 						countFound++
-						err = fileCallback(file)
-						if err != nil {
-							return err
+						callbackErr := fileCallback(file)
+						if callbackErr != nil {
+							return callbackErr
 						}
 					}
 				}

internal/caches/storage_file_test.go

@@ -19,6 +19,10 @@ import (
 )
 
 func TestFileStorage_Init(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -43,12 +47,16 @@ func TestFileStorage_Init(t *testing.T) {
 
 	time.Sleep(2 * time.Second)
 	storage.purgeLoop()
-	t.Log(storage.list.(*FileList).Stat(func(hash string) bool {
+	t.Log(storage.list.(*SQLiteFileList).Stat(func(hash string) bool {
 		return true
 	}))
 }
 
 func TestFileStorage_OpenWriter(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -96,6 +104,10 @@ func TestFileStorage_OpenWriter(t *testing.T) {
 }
 
 func TestFileStorage_OpenWriter_Partial(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   2,
 		IsOn: true,
@@ -134,6 +146,10 @@ func TestFileStorage_OpenWriter_Partial(t *testing.T) {
 }
 
 func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -202,6 +218,10 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
 }
 
 func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -231,7 +251,7 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
 
 			writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, -1, -1, false)
 			if err != nil {
-				if err != ErrFileIsWriting {
+				if errors.Is(err, ErrFileIsWriting) {
 					t.Error(err)
 					return
 				}
@@ -260,6 +280,10 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
 }
 
 func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -289,7 +313,7 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
 
 			writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, -1, -1, false)
 			if err != nil {
-				if err != ErrFileIsWriting {
+				if errors.Is(err, ErrFileIsWriting) {
 					t.Error(err)
 					return
 				}
@@ -319,6 +343,10 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
 }
 
 func TestFileStorage_Read(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -358,6 +386,10 @@ func TestFileStorage_Read(t *testing.T) {
 }
 
 func TestFileStorage_Read_HTTP_Response(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -414,6 +446,10 @@ func TestFileStorage_Read_HTTP_Response(t *testing.T) {
 }
 
 func TestFileStorage_Read_NotFound(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -450,6 +486,10 @@ func TestFileStorage_Read_NotFound(t *testing.T) {
 }
 
 func TestFileStorage_Delete(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -472,6 +512,10 @@ func TestFileStorage_Delete(t *testing.T) {
 }
 
 func TestFileStorage_Stat(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -500,6 +544,10 @@ func TestFileStorage_Stat(t *testing.T) {
 }
 
 func TestFileStorage_CleanAll(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -534,6 +582,10 @@ func TestFileStorage_CleanAll(t *testing.T) {
 }
 
 func TestFileStorage_Stop(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -552,6 +604,10 @@ func TestFileStorage_Stop(t *testing.T) {
 }
 
 func TestFileStorage_DecodeFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -571,6 +627,10 @@ func TestFileStorage_DecodeFile(t *testing.T) {
 }
 
 func TestFileStorage_RemoveCacheFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(nil)
 
 	defer storage.Stop()

internal/caches/storage_memory.go

@@ -1,6 +1,7 @@
 package caches
 
 import (
+	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -161,7 +162,7 @@ func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, h
 
 	// TODO 内存缓存暂时不支持分块内容存储
 	if isPartial {
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w (004)", ErrFileIsWriting)
 	}
 	return this.openWriter(key, expiredAt, status, headerSize, bodySize, maxSize, true)
 }
@@ -187,7 +188,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
 	var isWriting = false
 	_, ok := this.writingKeyMap[key]
 	if ok {
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w (005)", ErrFileIsWriting)
 	}
 	this.writingKeyMap[key] = zero.New()
 	defer func() {
@@ -208,7 +209,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
 			_ = this.list.Remove(hashString)
 			item = nil
 		} else {
-			return nil, ErrFileIsWriting
+			return nil, fmt.Errorf("%w (006)", ErrFileIsWriting)
 		}
 	}
 
@@ -366,7 +367,7 @@ func (this *MemoryStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy
 
 // CanUpdatePolicy 检查策略是否可以更新
 func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) bool {
-	return true
+	return newPolicy != nil && newPolicy.Type == serverconfigs.CachePolicyStorageMemory
 }
 
 // AddToList 将缓存添加到列表
@@ -577,7 +578,7 @@ func (this *MemoryStorage) flushItem(key string) {
 		Type:       writer.ItemType(),
 		Key:        key,
 		Host:       ParseHost(key),
-		ExpiredAt:  item.ExpiresAt,
+		ExpiresAt:  item.ExpiresAt,
 		HeaderSize: writer.HeaderSize(),
 		BodySize:   writer.BodySize(),
 	})

internal/caches/storage_memory_test.go

@@ -3,6 +3,7 @@ package caches
 import (
 	"bytes"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/rands"
 	"runtime"
@@ -156,7 +157,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
 		storage.AddToList(&Item{
 			Key:       "abc",
 			BodySize:  5,
-			ExpiredAt: expiredAt,
+			ExpiresAt: expiredAt,
 		})
 	}
 	{
@@ -173,7 +174,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
 		storage.AddToList(&Item{
 			Key:       "abc1",
 			BodySize:  5,
-			ExpiredAt: expiredAt,
+			ExpiresAt: expiredAt,
 		})
 	}
 	stat, err := storage.Stat()
@@ -200,7 +201,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
 		storage.AddToList(&Item{
 			Key:       "abc",
 			BodySize:  5,
-			ExpiredAt: expiredAt,
+			ExpiresAt: expiredAt,
 		})
 	}
 	{
@@ -216,7 +217,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
 		storage.AddToList(&Item{
 			Key:       "abc1",
 			BodySize:  5,
-			ExpiredAt: expiredAt,
+			ExpiresAt: expiredAt,
 		})
 	}
 	err := storage.CleanAll()
@@ -243,7 +244,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
 		storage.AddToList(&Item{
 			Key:       "abc",
 			BodySize:  5,
-			ExpiredAt: expiredAt,
+			ExpiresAt: expiredAt,
 		})
 	}
 	{
@@ -259,7 +260,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
 		storage.AddToList(&Item{
 			Key:       "abc1",
 			BodySize:  5,
-			ExpiredAt: expiredAt,
+			ExpiresAt: expiredAt,
 		})
 	}
 	err := storage.Purge([]string{"abc", "abc1"}, "")
@@ -271,6 +272,10 @@ func TestMemoryStorage_Purge(t *testing.T) {
 }
 
 func TestMemoryStorage_Expire(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
 		MemoryAutoPurgeInterval: 5,
 	}, nil)
@@ -294,7 +299,7 @@ func TestMemoryStorage_Expire(t *testing.T) {
 		storage.AddToList(&Item{
 			Key:       key,
 			BodySize:  5,
-			ExpiredAt: expiredAt,
+			ExpiresAt: expiredAt,
 		})
 	}
 	time.Sleep(70 * time.Second)

internal/caches/writer_memory.go

@@ -48,6 +48,10 @@ func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64,
 				SharedFragmentMemoryPool.IncreaseNew()
 			}
 		}
+	} else {
+		if expectedBodySize > 0 {
+			valueItem.BodyValue = make([]byte, 0, expectedBodySize)
+		}
 	}
 	var w = &MemoryWriter{
 		storage:          memoryStorage,
@@ -125,7 +129,6 @@ func (this *MemoryWriter) Close() error {
 	// 需要在Locker之外
 	defer this.once.Do(func() {
 		this.endFunc(this.item)
-		this.item = nil // free memory
 	})
 
 	if this.item == nil {
@@ -164,7 +167,6 @@ func (this *MemoryWriter) Discard() error {
 	// 需要在Locker之外
 	defer this.once.Do(func() {
 		this.endFunc(this.item)
-		this.item = nil // free memory
 	})
 
 	this.storage.locker.Lock()

internal/caches/writer_memory_test.go

@@ -0,0 +1,143 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"bytes"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"math/rand"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestNewMemoryWriter(t *testing.T) {
+	var storage = caches.NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
+		Id:          0,
+		IsOn:        false,
+		Name:        "",
+		Description: "",
+		Capacity: &shared.SizeCapacity{
+			Count: 8,
+			Unit:  shared.SizeCapacityUnitGB,
+		},
+	}, nil)
+	err := storage.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const size = 1 << 20
+	const chunkSize = 16 << 10
+	var data = bytes.Repeat([]byte{'A'}, chunkSize)
+
+	var before = time.Now()
+
+	var writer = caches.NewMemoryWriter(storage, "a", time.Now().Unix()+3600, 200, false, size, 1<<30, func(valueItem *caches.MemoryItem) {
+		t.Log(len(valueItem.BodyValue), "bytes")
+	})
+
+	for i := 0; i < size/chunkSize; i++ {
+		_, err = writer.Write(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
+}
+
+func BenchmarkMemoryWriter_Capacity(b *testing.B) {
+	b.ReportAllocs()
+
+	var storage = caches.NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
+		Id:          0,
+		IsOn:        false,
+		Name:        "",
+		Description: "",
+		Capacity: &shared.SizeCapacity{
+			Count: 8,
+			Unit:  shared.SizeCapacityUnitGB,
+		},
+	}, nil)
+	initErr := storage.Init()
+	if initErr != nil {
+		b.Fatal(initErr)
+	}
+
+	const size = 1 << 20
+	const chunkSize = 16 << 10
+	var data = bytes.Repeat([]byte{'A'}, chunkSize)
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var writer = caches.NewMemoryWriter(storage, "a"+strconv.Itoa(rand.Int()), time.Now().Unix()+3600, 200, false, size, 1<<30, func(valueItem *caches.MemoryItem) {
+			})
+
+			for i := 0; i < size/chunkSize; i++ {
+				_, err := writer.Write(data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+
+			err := writer.Close()
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
+func BenchmarkMemoryWriter_Capacity_Disabled(b *testing.B) {
+	b.ReportAllocs()
+
+	var storage = caches.NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
+		Id:          0,
+		IsOn:        false,
+		Name:        "",
+		Description: "",
+		Capacity: &shared.SizeCapacity{
+			Count: 8,
+			Unit:  shared.SizeCapacityUnitGB,
+		},
+	}, nil)
+	initErr := storage.Init()
+	if initErr != nil {
+		b.Fatal(initErr)
+	}
+
+	const size = 1 << 20
+	const chunkSize = 16 << 10
+	var data = bytes.Repeat([]byte{'A'}, chunkSize)
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var writer = caches.NewMemoryWriter(storage, "a"+strconv.Itoa(rand.Int()), time.Now().Unix()+3600, 200, false, 0, 1<<30, func(valueItem *caches.MemoryItem) {
+			})
+
+			for i := 0; i < size/chunkSize; i++ {
+				_, err := writer.Write(data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+
+			err := writer.Close()
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}

internal/compressions/reader_brotli_test.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"github.com/TeaOSLab/EdgeNode/internal/compressions"
 	"io"
+	"os"
 	"testing"
 )
 
@@ -49,3 +50,45 @@ func TestBrotliReader(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkBrotliReader(b *testing.B) {
+	data, err := os.ReadFile("./reader_brotli.go")
+	if err != nil {
+		b.Fatal(err)
+	}
+	var buf = bytes.NewBuffer([]byte{})
+	writer, err := compressions.NewBrotliWriter(buf, 5)
+	if err != nil {
+		b.Fatal(err)
+	}
+	_, err = writer.Write(data)
+	err = writer.Close()
+	if err != nil {
+		b.Fatal(err)
+	}
+	var compressedData = buf.Bytes()
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			reader, readerErr := compressions.NewBrotliReader(bytes.NewBuffer(compressedData))
+			if readerErr != nil {
+				b.Fatal(readerErr)
+			}
+			var readBuf = make([]byte, 1024)
+			for {
+				_, readErr := reader.Read(readBuf)
+				if readErr != nil {
+					if readErr != io.EOF {
+						b.Fatal(readErr)
+					}
+					break
+				}
+			}
+			closeErr := reader.Close()
+			if closeErr != nil {
+				b.Fatal(closeErr)
+			}
+		}
+	})
+}

internal/compressions/utils.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"io"
+	"net/http"
 )
 
 type ContentEncoding = string
@@ -58,3 +59,32 @@ func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType,
 	}
 	return nil, errors.New("invalid compression type '" + compressType + "'")
 }
+
+// SupportEncoding 检查是否支持某个编码
+func SupportEncoding(encoding string) bool {
+	return encoding == ContentEncodingBr ||
+		encoding == ContentEncodingGzip ||
+		encoding == ContentEncodingDeflate ||
+		encoding == ContentEncodingZSTD
+}
+
+// WrapHTTPResponse 包装http.Response对象
+func WrapHTTPResponse(resp *http.Response) {
+	if resp == nil {
+		return
+	}
+
+	var contentEncoding = resp.Header.Get("Content-Encoding")
+	if len(contentEncoding) == 0 || !SupportEncoding(contentEncoding) {
+		return
+	}
+
+	reader, err := NewReader(resp.Body, contentEncoding)
+	if err != nil {
+		// unable to decode, we ignore the error
+		return
+	}
+	resp.Header.Del("Content-Encoding")
+	resp.Header.Del("Content-Length")
+	resp.Body = reader
+}

internal/compressions/writer_gzip_test.go

@@ -5,12 +5,15 @@ package compressions_test
 import (
 	"bytes"
 	"github.com/TeaOSLab/EdgeNode/internal/compressions"
-	"strings"
 	"testing"
 )
 
 func BenchmarkGzipWriter_Write(b *testing.B) {
-	var data = []byte(strings.Repeat("A", 1024))
+	var data = make([]byte, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = 'A' + byte(i%26)
+	}
+	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
 		var buf = &bytes.Buffer{}
@@ -36,7 +39,12 @@ func BenchmarkGzipWriter_Write(b *testing.B) {
 }
 
 func BenchmarkGzipWriter_Write_Parallel(b *testing.B) {
-	var data = []byte(strings.Repeat("A", 1024))
+	var data = make([]byte, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = 'A' + byte(i%26)
+	}
+
+	b.ResetTimer()
 
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {

internal/configs/api_config.go

@@ -12,8 +12,8 @@ const oldConfigFileName = "api.yaml"
 
 type APIConfig struct {
 	OldRPC struct {
-		Endpoints     []string `yaml:"endpoints" json:"endpoints"`
-		DisableUpdate bool     `yaml:"disableUpdate" json:"disableUpdate"`
+		Endpoints     []string `yaml:"endpoints,omitempty" json:"endpoints"`
+		DisableUpdate bool     `yaml:"disableUpdate,omitempty" json:"disableUpdate"`
 	} `yaml:"rpc,omitempty" json:"rpc"`
 
 	RPCEndpoints     []string `yaml:"rpc.endpoints,flow" json:"rpc.endpoints"`

internal/configs/cluster_config_test.go

@@ -4,11 +4,16 @@ package configs_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/configs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"gopkg.in/yaml.v3"
 	"testing"
 )
 
 func TestLoadClusterConfig(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	config, err := configs.LoadClusterConfig()
 	if err != nil {
 		t.Fatal(err)

internal/const/const.go

@@ -1,7 +1,7 @@
 package teaconst
 
 const (
-	Version = "1.3.2"
+	Version = "1.3.4"
 
 	ProductName = "Edge Node"
 	ProcessName = "edge-node"
@@ -16,4 +16,6 @@ const (
 
 	AccessLogSockName    = "edge-node.accesslog"
 	CacheGarbageSockName = "edge-node.cache.garbage"
+
+	EnableKVCacheStore = false // determine store cache keys in KVStore or sqlite
 )

internal/firewalls/firewall.go

@@ -6,8 +6,10 @@ import (
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"os"
 	"runtime"
 	"sync"
+	"time"
 )
 
 var currentFirewall FirewallInterface
@@ -35,6 +37,23 @@ func Firewall() FirewallInterface {
 		return currentFirewall
 	}
 
+	// http firewall
+	{
+		endpoint, _ := os.LookupEnv("EDGE_HTTP_FIREWALL_ENDPOINT")
+		if len(endpoint) > 0 {
+			var httpFirewall = NewHTTPFirewall(endpoint)
+			for i := 0; i < 10; i++ {
+				if httpFirewall.IsReady() {
+					currentFirewall = httpFirewall
+					remotelogs.Println("FIREWALL", "using http firewall '"+endpoint+"'")
+					break
+				}
+				time.Sleep(1 * time.Second)
+			}
+			return httpFirewall
+		}
+	}
+
 	// nftables
 	if runtime.GOOS == "linux" {
 		nftables, err := NewNFTablesFirewall()

internal/firewalls/firewall_http.go

@@ -0,0 +1,150 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package firewalls
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+type HTTPFirewall struct {
+	client   *http.Client
+	endpoint string
+}
+
+func NewHTTPFirewall(endpoint string) *HTTPFirewall {
+	return &HTTPFirewall{
+		client:   http.DefaultClient,
+		endpoint: endpoint,
+	}
+}
+
+// Name 名称
+func (this *HTTPFirewall) Name() string {
+	result, err := this.get("/name", nil)
+	if err != nil {
+		return ""
+	}
+	return result.GetString("name")
+}
+
+// IsReady 是否已准备被调用
+func (this *HTTPFirewall) IsReady() bool {
+	result, err := this.get("/isReady", nil)
+	if err != nil {
+		return false
+	}
+	return result.GetBool("result")
+}
+
+// IsMock 是否为模拟
+func (this *HTTPFirewall) IsMock() bool {
+	result, err := this.get("/isMock", nil)
+	if err != nil {
+		return false
+	}
+	return result.GetBool("result")
+}
+
+// AllowPort 允许端口
+func (this *HTTPFirewall) AllowPort(port int, protocol string) error {
+	_, err := this.get("/allowPort", map[string]string{
+		"port":     types.String(port),
+		"protocol": protocol,
+	})
+	return err
+}
+
+// RemovePort 删除端口
+func (this *HTTPFirewall) RemovePort(port int, protocol string) error {
+	_, err := this.get("/removePort", map[string]string{
+		"port":     types.String(port),
+		"protocol": protocol,
+	})
+	return err
+}
+
+// RejectSourceIP 拒绝某个源IP连接
+func (this *HTTPFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
+	_, err := this.get("/rejectSourceIP", map[string]string{
+		"ip":             ip,
+		"timeoutSeconds": types.String(timeoutSeconds),
+	})
+	return err
+}
+
+// DropSourceIP 丢弃某个源IP数据
+// ip 要封禁的IP
+// timeoutSeconds 过期时间
+// async 是否异步
+func (this *HTTPFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
+	var asyncString = "false"
+	if async {
+		asyncString = "true"
+	}
+	_, err := this.get("/dropSourceIP", map[string]string{
+		"ip":             ip,
+		"timeoutSeconds": types.String(timeoutSeconds),
+		"async":          asyncString,
+	})
+	return err
+}
+
+// RemoveSourceIP 删除某个源IP
+func (this *HTTPFirewall) RemoveSourceIP(ip string) error {
+	_, err := this.get("/removeSourceIP", map[string]string{
+		"ip": ip,
+	})
+	return err
+}
+
+func (this *HTTPFirewall) get(path string, args map[string]string) (result maps.Map, err error) {
+	var urlString = this.endpoint + path
+	if len(args) > 0 {
+		var query = &url.Values{}
+		for k, v := range args {
+			query.Add(k, v)
+		}
+		urlString += "?" + query.Encode()
+	}
+	req, err := http.NewRequest(http.MethodGet, urlString, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := this.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("server response code '" + types.String(resp.StatusCode) + "'")
+	}
+
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response failed: %w", err)
+	}
+
+	if len(data) == 0 {
+		return maps.Map{}, nil
+	}
+
+	result = maps.Map{}
+	err = json.Unmarshal(data, &result)
+	if err != nil {
+		return nil, fmt.Errorf("decode response failed: %w", err)
+	}
+
+	return
+}

internal/firewalls/nftables/conn_test.go

@@ -7,7 +7,6 @@ package nftables_test
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
 	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
-	"os/exec"
 	"testing"
 )
 
@@ -21,7 +20,10 @@ func TestConn_Test(t *testing.T) {
 }
 
 func TestConn_GetTable_NotFound(t *testing.T) {
-	var conn = nftables.NewConn()
+	conn, err := nftables.NewConn()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	table, err := conn.GetTable("a", nftables.TableFamilyIPv4)
 	if err != nil {
@@ -36,7 +38,10 @@ func TestConn_GetTable_NotFound(t *testing.T) {
 }
 
 func TestConn_GetTable(t *testing.T) {
-	var conn = nftables.NewConn()
+	conn, err := nftables.NewConn()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	table, err := conn.GetTable("myFilter", nftables.TableFamilyIPv4)
 	if err != nil {
@@ -51,7 +56,10 @@ func TestConn_GetTable(t *testing.T) {
 }
 
 func TestConn_AddTable(t *testing.T) {
-	var conn = nftables.NewConn()
+	conn, err := nftables.NewConn()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	{
 		table, err := conn.AddIPv4Table("test_ipv4")
@@ -70,8 +78,12 @@ func TestConn_AddTable(t *testing.T) {
 }
 
 func TestConn_DeleteTable(t *testing.T) {
-	var conn = nftables.NewConn()
-	err := conn.DeleteTable("test_ipv4", nftables.TableFamilyIPv4)
+	conn, err := nftables.NewConn()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = conn.DeleteTable("test_ipv4", nftables.TableFamilyIPv4)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/iplibrary/action_http_api_test.go

@@ -3,11 +3,16 @@ package iplibrary
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 )
 
 func TestHTTPAPIAction_AddItem(t *testing.T) {
-	action := NewHTTPAPIAction()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var action = NewHTTPAPIAction()
 	action.config = &firewallconfigs.FirewallActionHTTPAPIConfig{
 		URL:            "http://127.0.0.1:2345/post",
 		TimeoutSeconds: 0,
@@ -24,7 +29,11 @@ func TestHTTPAPIAction_AddItem(t *testing.T) {
 }
 
 func TestHTTPAPIAction_DeleteItem(t *testing.T) {
-	action := NewHTTPAPIAction()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var action = NewHTTPAPIAction()
 	action.config = &firewallconfigs.FirewallActionHTTPAPIConfig{
 		URL:            "http://127.0.0.1:2345/post",
 		TimeoutSeconds: 0,

internal/iplibrary/action_ipset_test.go

@@ -4,13 +4,19 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"github.com/iwind/TeaGo/maps"
 	"testing"
 	"time"
 )
 
 func TestIPSetAction_Init(t *testing.T) {
-	action := iplibrary.NewIPSetAction()
+	_, lookupErr := executils.LookPath("iptables")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = iplibrary.NewIPSetAction()
 	err := action.Init(&firewallconfigs.FirewallActionConfig{
 		Params: maps.Map{
 			"path":      "/usr/bin/iptables",
@@ -25,6 +31,11 @@ func TestIPSetAction_Init(t *testing.T) {
 }
 
 func TestIPSetAction_AddItem(t *testing.T) {
+	_, lookupErr := executils.LookPath("iptables")
+	if lookupErr != nil {
+		return
+	}
+
 	var action = iplibrary.NewIPSetAction()
 	action.SetConfig(&firewallconfigs.FirewallActionIPSetConfig{
 		Path:          "/usr/bin/iptables",
@@ -84,7 +95,12 @@ func TestIPSetAction_AddItem(t *testing.T) {
 }
 
 func TestIPSetAction_DeleteItem(t *testing.T) {
-	action := iplibrary.NewIPSetAction()
+	_, lookupErr := executils.LookPath("firewalld")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = iplibrary.NewIPSetAction()
 	err := action.Init(&firewallconfigs.FirewallActionConfig{
 		Params: maps.Map{
 			"path":      "/usr/bin/firewalld",

internal/iplibrary/action_iptables_test.go

@@ -3,12 +3,18 @@ package iplibrary
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"testing"
 	"time"
 )
 
 func TestIPTablesAction_AddItem(t *testing.T) {
-	action := NewIPTablesAction()
+	_, lookupErr := executils.LookPath("iptables")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = NewIPTablesAction()
 	action.config = &firewallconfigs.FirewallActionIPTablesConfig{
 		Path: "/usr/bin/iptables",
 	}
@@ -40,7 +46,12 @@ func TestIPTablesAction_AddItem(t *testing.T) {
 }
 
 func TestIPTablesAction_DeleteItem(t *testing.T) {
-	action := NewIPTablesAction()
+	_, lookupErr := executils.LookPath("firewalld")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = NewIPTablesAction()
 	action.config = &firewallconfigs.FirewallActionIPTablesConfig{
 		Path: "/usr/bin/firewalld",
 	}

internal/iplibrary/action_manager_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestActionManager_UpdateActions(t *testing.T) {
-	manager := NewActionManager()
+	var manager = NewActionManager()
 	manager.UpdateActions([]*firewallconfigs.FirewallActionConfig{
 		{
 			Id:   1,

internal/iplibrary/action_script_test.go

@@ -3,11 +3,16 @@ package iplibrary
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 	"time"
 )
 
 func TestScriptAction_AddItem(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	action := NewScriptAction()
 	action.config = &firewallconfigs.FirewallActionScriptConfig{
 		Path: "/tmp/ip-item.sh",
@@ -27,6 +32,10 @@ func TestScriptAction_AddItem(t *testing.T) {
 }
 
 func TestScriptAction_DeleteItem(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	action := NewScriptAction()
 	action.config = &firewallconfigs.FirewallActionScriptConfig{
 		Path: "/tmp/ip-item.sh",

internal/iplibrary/ip_item_test.go

@@ -2,6 +2,7 @@ package iplibrary
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
 	"runtime"
 	"testing"
@@ -75,8 +76,14 @@ func TestIPItem_Contains(t *testing.T) {
 }
 
 func TestIPItem_Memory(t *testing.T) {
+	var isSingleTest = testutils.IsSingleTesting()
+
 	var list = NewIPList()
-	for i := 0; i < 2_000_000; i ++ {
+	var count = 100
+	if isSingleTest {
+		count = 2_000_000
+	}
+	for i := 0; i < count; i++ {
 		list.Add(&IPItem{
 			Type:       "ip",
 			Id:         uint64(i),
@@ -87,7 +94,9 @@ func TestIPItem_Memory(t *testing.T) {
 		})
 	}
 	t.Log("waiting")
-	time.Sleep(10 * time.Second)
+	if isSingleTest {
+		time.Sleep(10 * time.Second)
+	}
 }
 
 func BenchmarkIPItem_Contains(b *testing.B) {
@@ -105,4 +114,3 @@ func BenchmarkIPItem_Contains(b *testing.B) {
 		}
 	}
 }
-

internal/iplibrary/ip_list_db_test.go

@@ -16,6 +16,9 @@ func TestIPListDB_AddItem(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	err = db.AddItem(&pb.IPItem{
 		Id:                            1,
@@ -60,6 +63,9 @@ func TestIPListDB_ReadItems(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	defer func() {
 		_ = db.Close()
@@ -77,6 +83,9 @@ func TestIPListDB_ReadMaxVersion(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 	t.Log(db.ReadMaxVersion())
 }
 
@@ -85,6 +94,10 @@ func TestIPListDB_UpdateMaxVersion(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
+
 	err = db.UpdateMaxVersion(1027)
 	if err != nil {
 		t.Fatal(err)

internal/iplibrary/list_utils_test.go

@@ -3,12 +3,17 @@
 package iplibrary
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 	"time"
 )
 
 func TestIPIsAllowed(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var manager = NewIPListManager()
 	manager.init()
 
 	var before = time.Now()

internal/iplibrary/manager_ip_list_test.go

@@ -2,13 +2,18 @@ package iplibrary
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/logs"
 	"testing"
 	"time"
 )
 
 func TestIPListManager_init(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var manager = NewIPListManager()
 	manager.init()
 	t.Log(manager.listMap)
 	t.Log(SharedServerListManager.blackMap)
@@ -16,7 +21,11 @@ func TestIPListManager_init(t *testing.T) {
 }
 
 func TestIPListManager_check(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var manager = NewIPListManager()
 	manager.init()
 
 	var before = time.Now()
@@ -28,7 +37,11 @@ func TestIPListManager_check(t *testing.T) {
 }
 
 func TestIPListManager_loop(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var  manager = NewIPListManager()
 	manager.Start()
 	err := manager.loop()
 	if err != nil {

internal/monitor/value_queue_test.go

@@ -5,6 +5,7 @@ package monitor
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/logs"
 	"google.golang.org/grpc/status"
@@ -12,6 +13,10 @@ import (
 )
 
 func TestValueQueue_RPC(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	rpcClient, err := rpc.SharedRPC()
 	if err != nil {
 		t.Fatal(err)

internal/nodes/api_stream.go

@@ -209,7 +209,7 @@ func (this *APIStream) handleWriteCache(message *pb.NodeStreamMessage) error {
 	storage.AddToList(&caches.Item{
 		Type:       writer.ItemType(),
 		Key:        msg.Key,
-		ExpiredAt:  expiredAt,
+		ExpiresAt:  expiredAt,
 		HeaderSize: writer.HeaderSize(),
 		BodySize:   writer.BodySize(),
 	})

internal/nodes/api_stream_test.go

@@ -1,8 +1,15 @@
 package nodes
 
-import "testing"
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"testing"
+)
 
 func TestAPIStream_Start(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	apiStream := NewAPIStream()
 	apiStream.Start()
 }

internal/nodes/http_access_log_queue.go

@@ -67,7 +67,8 @@ func (this *HTTPAccessLogQueue) Push(accessLog *pb.HTTPAccessLog) {
 
 // 上传访问日志
 func (this *HTTPAccessLogQueue) loop() error {
-	var accessLogs = []*pb.HTTPAccessLog{}
+	const maxLen = 2000
+	var accessLogs = make([]*pb.HTTPAccessLog, 0, maxLen)
 	var count = 0
 
 Loop:
@@ -78,7 +79,7 @@ Loop:
 			count++
 
 			// 每次只提交 N 条访问日志，防止网络拥堵
-			if count > 2000 {
+			if count >= maxLen {
 				break Loop
 			}
 		default:

internal/nodes/http_access_log_queue_test.go

@@ -110,6 +110,10 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
 }
 
 func TestHTTPAccessLogQueue_Memory(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	testutils.StartMemoryStats(t)
 
 	debug.SetGCPercent(10)
@@ -174,3 +178,55 @@ func BenchmarkHTTPAccessLogQueue_ToValidUTF8String(b *testing.B) {
 		_ = strings.ToValidUTF8(s, "")
 	}
 }
+
+func BenchmarkAppendAccessLogs(b *testing.B) {
+	b.ReportAllocs()
+
+	var stat1 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat1)
+
+	const count = 20000
+	var a = make([]*pb.HTTPAccessLog, 0, count)
+	for i := 0; i < b.N; i++ {
+		a = append(a, &pb.HTTPAccessLog{
+			RequestPath: "/hello/world",
+			Host:        "example.com",
+			RequestBody: bytes.Repeat([]byte{'A'}, 1024),
+		})
+		if len(a) == count {
+			a = make([]*pb.HTTPAccessLog, 0, count)
+		}
+	}
+
+	_ = len(a)
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+	b.Log((stat2.TotalAlloc-stat1.TotalAlloc)>>20, "MB")
+}
+
+func BenchmarkAppendAccessLogs2(b *testing.B) {
+	b.ReportAllocs()
+
+	var stat1 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat1)
+
+	const count = 20000
+	var a = []*pb.HTTPAccessLog{}
+	for i := 0; i < b.N; i++ {
+		a = append(a, &pb.HTTPAccessLog{
+			RequestPath: "/hello/world",
+			Host:        "example.com",
+			RequestBody: bytes.Repeat([]byte{'A'}, 1024),
+		})
+		if len(a) == count {
+			a = []*pb.HTTPAccessLog{}
+		}
+	}
+
+	_ = len(a)
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+	b.Log((stat2.TotalAlloc-stat1.TotalAlloc)>>20, "MB")
+}

internal/nodes/http_client_pool_test.go

@@ -3,13 +3,14 @@ package nodes
 import (
 	"context"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"runtime"
 	"testing"
 	"time"
 )
 
 func TestHTTPClientPool_Client(t *testing.T) {
-	pool := NewHTTPClientPool()
+	var pool = NewHTTPClientPool()
 
 	{
 		var origin = &serverconfigs.OriginConfig{
@@ -54,7 +55,10 @@ func TestHTTPClientPool_cleanClients(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		t.Log("get", i)
 		_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil, false)
-		time.Sleep(1 * time.Second)
+
+		if testutils.IsSingleTesting() {
+			time.Sleep(1 * time.Second)
+		}
 	}
 }
 

internal/nodes/http_request.go

@@ -102,6 +102,8 @@ type HTTPRequest struct {
 	disableLog bool // 是否在当前请求中关闭Log
 	forceLog   bool // 是否强制记录日志
 
+	isHijacked bool
+
 	// script相关操作
 	isDone bool
 }
@@ -193,17 +195,35 @@ func (this *HTTPRequest) Do() {
 		}
 
 		// 套餐
-		if this.ReqServer.UserPlan != nil && !this.ReqServer.UserPlan.IsAvailable() {
-			this.doPlanExpires()
-			this.doEnd()
-			return
+		if this.ReqServer.UserPlan != nil {
+			if this.doPlanBefore() {
+				this.doEnd()
+				return
+			}
 		}
 
 		// 流量限制
 		if this.ReqServer.TrafficLimitStatus != nil && this.ReqServer.TrafficLimitStatus.IsValid() {
-			this.doTrafficLimit()
-			this.doEnd()
-			return
+			if this.doTrafficLimit(this.ReqServer.TrafficLimitStatus) {
+				this.doEnd()
+				return
+			}
+		}
+
+		// UAM
+		var uamIsCalled = false
+		if !this.isHealthCheck {
+			if this.web.UAM == nil && this.ReqServer.UAM != nil && this.ReqServer.UAM.IsOn {
+				this.web.UAM = this.ReqServer.UAM
+			}
+
+			if this.web.UAM != nil && this.web.UAM.IsOn && this.isUAMRequest() {
+				uamIsCalled = true
+				if this.doUAM() {
+					this.doEnd()
+					return
+				}
+			}
 		}
 
 		// WAF
@@ -215,16 +235,8 @@ func (this *HTTPRequest) Do() {
 		}
 
 		// UAM
-		if !this.isHealthCheck {
-			if this.web.UAM != nil {
-				if this.web.UAM.IsOn {
-					if this.doUAM() {
-						this.doEnd()
-						return
-					}
-				}
-			} else if this.ReqServer.UAM != nil && this.ReqServer.UAM.IsOn {
-				this.web.UAM = this.ReqServer.UAM
+		if !this.isHealthCheck && !uamIsCalled {
+			if this.web.UAM != nil && this.web.UAM.IsOn {
 				if this.doUAM() {
 					this.doEnd()
 					return
@@ -280,6 +292,16 @@ func (this *HTTPRequest) Do() {
 		if this.web.Compression != nil && this.web.Compression.IsOn && this.web.Compression.Level > 0 {
 			this.writer.SetCompression(this.web.Compression)
 		}
+
+		// HLS
+		if this.web.HLS != nil &&
+			this.web.HLS.Encrypting != nil &&
+			this.web.HLS.Encrypting.IsOn {
+			if this.processHLSBefore() {
+				this.doEnd()
+				return
+			}
+		}
 	}
 
 	// 开始调用
@@ -631,6 +653,11 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
 		this.web.CC = web.CC
 	}
 
+	// HLS
+	if web.HLS != nil && (web.HLS.IsPrior || isTop) {
+		this.web.HLS = web.HLS
+	}
+
 	// 重写规则
 	if len(web.RewriteRefs) > 0 {
 		for index, ref := range web.RewriteRefs {
@@ -1430,11 +1457,25 @@ func (this *HTTPRequest) requestScheme() string {
 
 // 请求的服务器地址中的端口
 func (this *HTTPRequest) requestServerPort() int {
-	_, port, err := net.SplitHostPort(this.ServerAddr)
-	if err == nil {
-		return types.Int(port)
+	if len(this.ServerAddr) > 0 {
+		_, port, err := net.SplitHostPort(this.ServerAddr)
+		if err == nil && len(port) > 0 {
+			return types.Int(port)
+		}
 	}
-	return 0
+
+	var host = this.RawReq.Host
+	if len(host) > 0 {
+		_, port, err := net.SplitHostPort(host)
+		if err == nil && len(port) > 0 {
+			return types.Int(port)
+		}
+	}
+
+	if this.IsHTTP {
+		return 80
+	}
+	return 443
 }
 
 func (this *HTTPRequest) Id() string {

internal/nodes/http_request_auth.go

@@ -43,7 +43,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
 			if uriChanged {
 				this.uri = newURI
 			}
-			this.tags = append(this.tags, ref.AuthPolicy.Type)
+			this.tags = append(this.tags, "auth:"+ref.AuthPolicy.Type)
 			return
 		} else {
 			// Basic Auth比较特殊
@@ -64,7 +64,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
 				}
 			}
 			this.writer.WriteHeader(http.StatusUnauthorized)
-			this.tags = append(this.tags, ref.AuthPolicy.Type)
+			this.tags = append(this.tags, "auth:"+ref.AuthPolicy.Type)
 			return true
 		}
 	}

internal/nodes/http_request_hls.go

@@ -0,0 +1,16 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build !plus
+
+package nodes
+
+import "net/http"
+
+func (this *HTTPRequest) processHLSBefore() (blocked bool) {
+	//  stub
+	return false
+}
+
+func (this *HTTPRequest) processM3u8Response(resp *http.Response) error {
+	// stub
+	return nil
+}

internal/nodes/http_request_mismatch.go

@@ -34,57 +34,78 @@ func (this *HTTPRequest) doMismatch() {
 	}
 
 	// 根据配置进行相应的处理
-	var globalServerConfig = sharedNodeConfig.GlobalServerConfig
-	if globalServerConfig != nil && globalServerConfig.HTTPAll.MatchDomainStrictly {
-		var statusCode = 404
-		var httpAllConfig = globalServerConfig.HTTPAll
-		var mismatchAction = httpAllConfig.DomainMismatchAction
+	var nodeConfig = sharedNodeConfig // copy
+	if nodeConfig != nil {
+		var globalServerConfig = nodeConfig.GlobalServerConfig
+		if globalServerConfig != nil && globalServerConfig.HTTPAll.MatchDomainStrictly {
+			var statusCode = 404
+			var httpAllConfig = globalServerConfig.HTTPAll
+			var mismatchAction = httpAllConfig.DomainMismatchAction
 
-		if mismatchAction != nil && mismatchAction.Options != nil {
-			var mismatchStatusCode = mismatchAction.Options.GetInt("statusCode")
-			if mismatchStatusCode > 0 && mismatchStatusCode >= 100 && mismatchStatusCode < 1000 {
-				statusCode = mismatchStatusCode
-			}
-		}
-
-		// 是否正在访问IP
-		if globalServerConfig.HTTPAll.NodeIPShowPage && utils.IsWildIP(this.ReqHost) {
-			this.writer.statusCode = statusCode
-			var contentHTML = this.Format(globalServerConfig.HTTPAll.NodeIPPageHTML)
-			this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
-			this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
-			this.writer.WriteHeader(statusCode)
-			_, _ = this.writer.WriteString(contentHTML)
-			return
-		}
-
-		// 检查cc
-		// TODO 可以在管理端配置是否开启以及最多尝试次数
-		// 要考虑到服务在切换集群时，域名未生效状态时，用户访问的仍然是老集群中的节点，就会产生找不到域名的情况
-		if len(remoteIP) > 0 {
-			const maxAttempts = 100
-			if ttlcache.SharedInt64Cache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
-				// 在加入之前再次检查黑名单
-				if !waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
-					waf.SharedIPBlackList.Add(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP, time.Now().Unix()+3600)
+			if mismatchAction != nil && mismatchAction.Options != nil {
+				var mismatchStatusCode = mismatchAction.Options.GetInt("statusCode")
+				if mismatchStatusCode > 0 && mismatchStatusCode >= 100 && mismatchStatusCode < 1000 {
+					statusCode = mismatchStatusCode
 				}
 			}
-		}
 
-		// 处理当前连接
-		if mismatchAction != nil && mismatchAction.Code == serverconfigs.DomainMismatchActionPage {
-			if mismatchAction.Options != nil {
+			// 是否正在访问IP
+			if globalServerConfig.HTTPAll.NodeIPShowPage && utils.IsWildIP(this.ReqHost) {
 				this.writer.statusCode = statusCode
-				var contentHTML = this.Format(mismatchAction.Options.GetString("contentHTML"))
+				var contentHTML = this.Format(globalServerConfig.HTTPAll.NodeIPPageHTML)
 				this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 				this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
 				this.writer.WriteHeader(statusCode)
-				_, _ = this.writer.Write([]byte(contentHTML))
-			} else {
-				http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
+				_, _ = this.writer.WriteString(contentHTML)
+				return
 			}
-			return
-		} else {
+
+			// 检查cc
+			// TODO 可以在管理端配置是否开启以及最多尝试次数
+			// 要考虑到服务在切换集群时，域名未生效状态时，用户访问的仍然是老集群中的节点，就会产生找不到域名的情况
+			if len(remoteIP) > 0 {
+				const maxAttempts = 100
+				if ttlcache.SharedInt64Cache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
+					// 在加入之前再次检查黑名单
+					if !waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
+						waf.SharedIPBlackList.Add(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP, time.Now().Unix()+3600)
+					}
+				}
+			}
+
+			// 处理当前连接
+			if mismatchAction != nil {
+				if mismatchAction.Code == serverconfigs.DomainMismatchActionPage {
+					if mismatchAction.Options != nil {
+						this.writer.statusCode = statusCode
+						var contentHTML = this.Format(mismatchAction.Options.GetString("contentHTML"))
+						this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+						this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
+						this.writer.WriteHeader(statusCode)
+						_, _ = this.writer.Write([]byte(contentHTML))
+					} else {
+						http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
+					}
+					return
+				}
+
+				if mismatchAction.Code == serverconfigs.DomainMismatchActionRedirect {
+					var url = this.Format(mismatchAction.Options.GetString("url"))
+					if len(url) > 0 {
+						httpRedirect(this.writer, this.RawReq, url, http.StatusTemporaryRedirect)
+					} else {
+						http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
+					}
+					return
+				}
+
+				if mismatchAction.Code == serverconfigs.DomainMismatchActionClose {
+					http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
+					this.Close()
+					return
+				}
+			}
+
 			http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
 			this.Close()
 			return

internal/nodes/http_request_page.go

@@ -151,7 +151,7 @@ func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, sta
 				}
 				return true
 			} else if page.BodyType == serverconfigs.HTTPPageBodyTypeRedirectURL {
-				var newURL = page.URL
+				var newURL = this.Format(page.URL)
 				if len(newURL) == 0 {
 					newURL = "/"
 				}

internal/nodes/http_request_plan_before.go

@@ -0,0 +1,10 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+//go:build !plus
+
+package nodes
+
+// 检查套餐
+func (this *HTTPRequest) doPlanBefore() (blocked bool) {
+	// stub
+	return false
+}

internal/nodes/http_request_plan_expires.go

@@ -1,19 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package nodes
-
-import (
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"net/http"
-)
-
-// 套餐过期
-func (this *HTTPRequest) doPlanExpires() {
-	this.tags = append(this.tags, "plan")
-
-	var statusCode = http.StatusNotFound
-	this.ProcessResponseHeaders(this.writer.Header(), statusCode)
-
-	this.writer.WriteHeader(statusCode)
-	_, _ = this.writer.WriteString(this.Format(serverconfigs.DefaultPlanExpireNoticePageBody))
-}

internal/nodes/http_request_reverse_proxy.go

@@ -8,6 +8,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/minifiers"
 	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"io"
@@ -93,8 +94,8 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 		if origin == nil {
 			origin = this.reverseProxy.NextOrigin(requestCall)
 			if origin != nil && origin.Id > 0 && (*failStatusCode >= 403 && *failStatusCode <= 404) && lists.ContainsInt64(failedOriginIds, origin.Id) {
-				this.writeCode(*failStatusCode, "", "")
-				return
+				shouldRetry = false
+				isLastRetry = true
 			}
 		}
 		requestCall.CallResponseCallbacks(this.writer)
@@ -381,11 +382,20 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 		return
 	}
 
+	if resp == nil {
+		this.write50x(requestErr, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
+		return
+	}
+
+	// fix Content-Type
+	if resp.Header["Content-Type"] == nil {
+		resp.Header["Content-Type"] = []string{}
+	}
+
 	// 40x && 50x
 	*failStatusCode = resp.StatusCode
-	if resp != nil &&
-		((resp.StatusCode >= 500 && resp.StatusCode < 510 && this.reverseProxy.Retry50X) ||
-			(resp.StatusCode >= 403 && resp.StatusCode <= 404 && this.reverseProxy.Retry40X)) &&
+	if ((resp.StatusCode >= 500 && resp.StatusCode < 510 && this.reverseProxy.Retry50X) ||
+		(resp.StatusCode >= 403 && resp.StatusCode <= 404 && this.reverseProxy.Retry40X)) &&
 		(originId > 0 || (lnNodeId > 0 && hasMultipleLnNodes)) &&
 		!isLastRetry {
 		if resp.Body != nil {
@@ -397,8 +407,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 	}
 
 	// 尝试从缓存中恢复
-	if resp != nil &&
-		resp.StatusCode >= 500 && // support 50X only
+	if resp.StatusCode >= 500 && // support 50X only
 		resp.StatusCode < 510 &&
 		this.cacheCanTryStale &&
 		this.web.Cache.Stale != nil &&
@@ -434,21 +443,43 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 
 	// Page optimization
 	if this.web.Optimization != nil && resp.Body != nil && this.cacheRef != nil /** must under cache **/ {
-		err := this.web.Optimization.FilterResponse(this.URL(), resp)
+		err := minifiers.MinifyResponse(this.web.Optimization, this.URL(), resp)
 		if err != nil {
-			this.write50x(err, http.StatusBadGateway, "Page Optimization: Fail to read content from origin", "内容优化：从源站读取内容失败", false)
+			this.write50x(err, http.StatusBadGateway, "Page Optimization: fail to read content from origin", "内容优化：从源站读取内容失败", false)
+			return
+		}
+	}
+
+	// HLS
+	if this.web.HLS != nil &&
+		this.web.HLS.Encrypting != nil &&
+		this.web.HLS.Encrypting.IsOn &&
+		resp.StatusCode == http.StatusOK {
+		m3u8Err := this.processM3u8Response(resp)
+		if m3u8Err != nil {
+			this.write50x(m3u8Err, http.StatusBadGateway, "m3u8 encrypt: fail to read content from origin", "m3u8加密：从源站读取内容失败", false)
 			return
 		}
 	}
 
 	// 设置Charset
-	// TODO 这里应该可以设置文本类型的列表，以及是否强制覆盖所有文本类型的字符集
+	// TODO 这里应该可以设置文本类型的列表
 	if this.web.Charset != nil && this.web.Charset.IsOn && len(this.web.Charset.Charset) > 0 {
 		contentTypes, ok := resp.Header["Content-Type"]
 		if ok && len(contentTypes) > 0 {
 			var contentType = contentTypes[0]
+			if this.web.Charset.Force {
+				var semiIndex = strings.Index(contentType, ";")
+				if semiIndex > 0 {
+					contentType = contentType[:semiIndex]
+				}
+			}
 			if _, found := textMimeMap[contentType]; found {
-				resp.Header["Content-Type"][0] = contentType + "; charset=" + this.web.Charset.Charset
+				var newCharset = this.web.Charset.Charset
+				if this.web.Charset.IsUpper {
+					newCharset = strings.ToUpper(newCharset)
+				}
+				resp.Header["Content-Type"][0] = contentType + "; charset=" + newCharset
 			}
 		}
 	}

internal/nodes/http_request_test.go

@@ -3,6 +3,7 @@ package nodes
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/iwind/TeaGo/assert"
+	"net/http"
 	"runtime"
 	"testing"
 )
@@ -10,26 +11,45 @@ import (
 func TestHTTPRequest_RedirectToHTTPS(t *testing.T) {
 	var a = assert.NewAssertion(t)
 	{
-		req := &HTTPRequest{
+		rawReq, err := http.NewRequest(http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var req = &HTTPRequest{
+			RawReq:    rawReq,
+			RawWriter: NewEmptyResponseWriter(nil),
 			ReqServer: &serverconfigs.ServerConfig{
+				IsOn: true,
 				Web: &serverconfigs.HTTPWebConfig{
+					IsOn:            true,
 					RedirectToHttps: &serverconfigs.HTTPRedirectToHTTPSConfig{},
 				},
 			},
 		}
+		req.init()
 		req.Do()
+
 		a.IsBool(req.web.RedirectToHttps.IsOn == false)
 	}
 	{
-		req := &HTTPRequest{
+		rawReq, err := http.NewRequest(http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var req = &HTTPRequest{
+			RawReq:    rawReq,
+			RawWriter: NewEmptyResponseWriter(nil),
 			ReqServer: &serverconfigs.ServerConfig{
+				IsOn: true,
 				Web: &serverconfigs.HTTPWebConfig{
+					IsOn: true,
 					RedirectToHttps: &serverconfigs.HTTPRedirectToHTTPSConfig{
 						IsOn: true,
 					},
 				},
 			},
 		}
+		req.init()
 		req.Do()
 		a.IsBool(req.web.RedirectToHttps.IsOn == true)
 	}

internal/nodes/http_request_traffic_limit.go

@@ -7,7 +7,19 @@ import (
 )
 
 // 流量限制
-func (this *HTTPRequest) doTrafficLimit() {
+func (this *HTTPRequest) doTrafficLimit(status *serverconfigs.TrafficLimitStatus) (blocked bool) {
+	if status == nil {
+		return false
+	}
+
+	// 如果是网站单独设置的流量限制，则检查是否已关闭
+	var config = this.ReqServer.TrafficLimit
+	if (config == nil || !config.IsOn) && status.PlanId == 0 {
+		return false
+	}
+
+	// 如果是套餐设置的流量限制，即使套餐变更了（变更套餐或者变更套餐的限制），仍然会提示流量超限
+
 	this.tags = append(this.tags, "trafficLimit")
 
 	var statusCode = 509
@@ -17,10 +29,19 @@ func (this *HTTPRequest) doTrafficLimit() {
 	this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 	this.writer.WriteHeader(statusCode)
 
-	var config = this.ReqServer.TrafficLimit
+	// check plan traffic limit
+	if (config == nil || !config.IsOn) && this.ReqServer.PlanId() > 0 && this.nodeConfig != nil {
+		var planConfig = this.nodeConfig.FindPlan(this.ReqServer.PlanId())
+		if planConfig != nil && planConfig.TrafficLimit != nil && planConfig.TrafficLimit.IsOn {
+			config = planConfig.TrafficLimit
+		}
+	}
+
 	if config != nil && len(config.NoticePageBody) != 0 {
 		_, _ = this.writer.WriteString(this.Format(config.NoticePageBody))
 	} else {
 		_, _ = this.writer.WriteString(this.Format(serverconfigs.DefaultTrafficLimitNoticePageBody))
 	}
+
+	return true
 }

internal/nodes/http_request_uam.go

@@ -4,7 +4,13 @@
 
 package nodes
 
-// UAM
-func (this *HTTPRequest) doUAM() (block bool) {
+func (this *HTTPRequest) isUAMRequest() bool {
+	// stub
+	return false
+}
+
+// UAM
+func (this *HTTPRequest) doUAM() (block bool) {
+	// stub
 	return false
 }

internal/nodes/http_request_waf.go

@@ -67,8 +67,8 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
 
 	// 当前服务的独立设置
 	if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
-		blocked, breakChecking := this.checkWAFRequest(this.web.FirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, false)
-		if blocked {
+		blockedRequest, breakChecking := this.checkWAFRequest(this.web.FirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, false)
+		if blockedRequest {
 			return true
 		}
 		if breakChecking {
@@ -78,8 +78,8 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
 
 	// 公用的防火墙设置
 	if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
-		blocked, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, this.web.FirewallRef.IgnoreGlobalRules)
-		if blocked {
+		blockedRequest, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, this.web.FirewallRef.IgnoreGlobalRules)
+		if blockedRequest {
 			return true
 		}
 		if breakChecking {
@@ -266,39 +266,42 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 		return
 	}
 
-	goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer, this.web.FirewallRef.DefaultCaptchaType)
-	if forceLog && logRequestBody && hasRequestBody && ruleSet != nil && ruleSet.HasAttackActions() {
+	result, err := w.MatchRequest(this, this.writer, this.web.FirewallRef.DefaultCaptchaType)
+	if result.IsAllowed && (len(result.AllowScope) == 0 || result.AllowScope == waf.AllowScopeGlobal) {
+		breakChecking = true
+	}
+	if forceLog && logRequestBody && result.HasRequestBody && result.Set != nil && result.Set.HasAttackActions() {
 		this.wafHasRequestBody = true
 	}
 	if err != nil {
 		if !this.canIgnore(err) {
-			remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
+			remotelogs.Warn("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
 		}
 		return
 	}
 
-	if ruleSet != nil {
+	if result.Set != nil {
 		if forceLog {
 			this.forceLog = true
 		}
 
-		if ruleSet.HasSpecialActions() {
+		if result.Set.HasSpecialActions() {
 			this.firewallPolicyId = firewallPolicy.Id
-			this.firewallRuleGroupId = types.Int64(ruleGroup.Id)
-			this.firewallRuleSetId = types.Int64(ruleSet.Id)
+			this.firewallRuleGroupId = types.Int64(result.Group.Id)
+			this.firewallRuleSetId = types.Int64(result.Set.Id)
 
-			if ruleSet.HasAttackActions() {
+			if result.Set.HasAttackActions() {
 				this.isAttack = true
 			}
 
 			// 添加统计
-			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
+			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, result.Set.Actions)
 		}
 
-		this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
+		this.firewallActions = append(result.Set.ActionCodes(), firewallPolicy.Mode)
 	}
 
-	return !goNext, false
+	return !result.GoNext, breakChecking
 }
 
 // call response waf
@@ -316,23 +319,26 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
 	}
 
 	if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
-		blocked = this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
-		if blocked {
+		blockedRequest, breakChecking := this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
+		if blockedRequest {
 			return true
 		}
+		if breakChecking {
+			return
+		}
 	}
 
 	// 公用的防火墙设置
 	if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
-		blocked = this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
-		if blocked {
+		blockedRequest, _ := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
+		if blockedRequest {
 			return true
 		}
 	}
 	return
 }
 
-func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, resp *http.Response, forceLog bool, logRequestBody bool, ignoreRules bool) (blocked bool) {
+func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, resp *http.Response, forceLog bool, logRequestBody bool, ignoreRules bool) (blocked bool, breakChecking bool) {
 	if firewallPolicy == nil || !firewallPolicy.IsOn || !firewallPolicy.Outbound.IsOn || firewallPolicy.Mode == firewallconfigs.FirewallModeBypass {
 		return
 	}
@@ -347,39 +353,42 @@ func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFi
 		return
 	}
 
-	goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchResponse(this, resp, this.writer)
-	if forceLog && logRequestBody && hasRequestBody && ruleSet != nil && ruleSet.HasAttackActions() {
+	result, err := w.MatchResponse(this, resp, this.writer)
+	if result.IsAllowed && (len(result.AllowScope) == 0 || result.AllowScope == waf.AllowScopeGlobal) {
+		breakChecking = true
+	}
+	if forceLog && logRequestBody && result.HasRequestBody && result.Set != nil && result.Set.HasAttackActions() {
 		this.wafHasRequestBody = true
 	}
 	if err != nil {
 		if !this.canIgnore(err) {
-			remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
+			remotelogs.Warn("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
 		}
 		return
 	}
 
-	if ruleSet != nil {
+	if result.Set != nil {
 		if forceLog {
 			this.forceLog = true
 		}
 
-		if ruleSet.HasSpecialActions() {
+		if result.Set.HasSpecialActions() {
 			this.firewallPolicyId = firewallPolicy.Id
-			this.firewallRuleGroupId = types.Int64(ruleGroup.Id)
-			this.firewallRuleSetId = types.Int64(ruleSet.Id)
+			this.firewallRuleGroupId = types.Int64(result.Group.Id)
+			this.firewallRuleSetId = types.Int64(result.Set.Id)
 
-			if ruleSet.HasAttackActions() {
+			if result.Set.HasAttackActions() {
 				this.isAttack = true
 			}
 
 			// 添加统计
-			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
+			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, result.Set.Actions)
 		}
 
-		this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
+		this.firewallActions = append(result.Set.ActionCodes(), firewallPolicy.Mode)
 	}
 
-	return !goNext
+	return !result.GoNext, breakChecking
 }
 
 // WAFRaw 原始请求

internal/nodes/http_writer.go

@@ -18,7 +18,6 @@ import (
 	setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
 	_ "github.com/biessek/golang-ico"
-	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"github.com/iwind/gowebp"
 	_ "golang.org/x/image/bmp"
@@ -340,7 +339,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 
 	cacheWriter, err := storage.OpenWriter(cacheKey, expiresAt, this.StatusCode(), this.calculateHeaderLength(), totalSize, cacheRef.MaxSizeBytes(), this.isPartial)
 	if err != nil {
-		if err == caches.ErrEntityTooLarge && addStatusHeader {
+		if errors.Is(err, caches.ErrEntityTooLarge) && addStatusHeader {
 			this.Header().Set("X-Cache", "BYPASS, entity too large")
 		}
 
@@ -494,7 +493,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 		return
 	}
 
-	var cacheReader = readers.NewTeeReaderCloser(resp.Body, this.cacheWriter)
+	var cacheReader = readers.NewTeeReaderCloser(resp.Body, this.cacheWriter, false)
 	resp.Body = cacheReader
 	this.rawReader = cacheReader
 
@@ -564,18 +563,18 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
 		}
 
 		var contentEncoding = this.GetHeader("Content-Encoding")
-		switch contentEncoding {
-		case "gzip", "deflate", "br", "zstd":
-			reader, err := compressions.NewReader(resp.Body, contentEncoding)
-			if err != nil {
+		if len(contentEncoding) > 0 {
+			if compressions.SupportEncoding(contentEncoding) {
+				reader, err := compressions.NewReader(resp.Body, contentEncoding)
+				if err != nil {
+					return
+				}
+				this.Header().Del("Content-Encoding")
+				this.Header().Del("Content-Length")
+				this.rawReader = reader
+			} else {
 				return
 			}
-			this.Header().Del("Content-Encoding")
-			this.Header().Del("Content-Length")
-			this.rawReader = reader
-		case "": // 空
-		default:
-			return
 		}
 
 		this.webpOriginContentType = contentType
@@ -603,7 +602,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
 	var contentEncoding = this.GetHeader("Content-Encoding")
 
 	if this.compressionConfig == nil || !this.compressionConfig.IsOn {
-		if lists.ContainsString([]string{"gzip", "deflate", "br", "zstd"}, contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
+		if compressions.SupportEncoding(contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
 			reader, err := compressions.NewReader(resp.Body, contentEncoding)
 			if err != nil {
 				return
@@ -625,7 +624,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
 	}
 
 	// 如果已经有编码则不处理
-	if len(contentEncoding) > 0 && (!this.compressionConfig.DecompressData || !lists.ContainsString([]string{"gzip", "deflate", "br", "zstd"}, contentEncoding)) {
+	if len(contentEncoding) > 0 && (!this.compressionConfig.DecompressData || !compressions.SupportEncoding(contentEncoding)) {
 		return
 	}
 
@@ -807,6 +806,8 @@ func (this *HTTPWriter) Write(data []byte) (n int, err error) {
 	}
 	n, err = this.writer.Write(data)
 
+	this.checkPlanBandwidth(n)
+
 	return
 }
 
@@ -967,6 +968,7 @@ func (this *HTTPWriter) Close() {
 func (this *HTTPWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
 	hijack, ok := this.rawWriter.(http.Hijacker)
 	if ok {
+		this.req.isHijacked = true
 		return hijack.Hijack()
 	}
 	return
@@ -1166,7 +1168,7 @@ func (this *HTTPWriter) finishWebP() {
 				this.cacheStorage.AddToList(&caches.Item{
 					Type:       webpCacheWriter.ItemType(),
 					Key:        webpCacheWriter.Key(),
-					ExpiredAt:  webpCacheWriter.ExpiredAt(),
+					ExpiresAt:  webpCacheWriter.ExpiredAt(),
 					StaleAt:    webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
 					HeaderSize: webpCacheWriter.HeaderSize(),
 					BodySize:   webpCacheWriter.BodySize(),
@@ -1205,7 +1207,7 @@ func (this *HTTPWriter) finishCache() {
 						this.cacheStorage.AddToList(&caches.Item{
 							Type:       this.cacheWriter.ItemType(),
 							Key:        this.cacheWriter.Key(),
-							ExpiredAt:  expiredAt,
+							ExpiresAt:  expiredAt,
 							StaleAt:    expiredAt + int64(this.calculateStaleLife()),
 							HeaderSize: this.cacheWriter.HeaderSize(),
 							BodySize:   this.cacheWriter.BodySize(),
@@ -1226,7 +1228,7 @@ func (this *HTTPWriter) finishCache() {
 					this.cacheStorage.AddToList(&caches.Item{
 						Type:       this.cacheWriter.ItemType(),
 						Key:        this.cacheWriter.Key(),
-						ExpiredAt:  expiredAt,
+						ExpiresAt:  expiredAt,
 						StaleAt:    expiredAt + int64(this.calculateStaleLife()),
 						HeaderSize: this.cacheWriter.HeaderSize(),
 						BodySize:   this.cacheWriter.BodySize(),
@@ -1249,7 +1251,7 @@ func (this *HTTPWriter) finishCompression() {
 				this.cacheStorage.AddToList(&caches.Item{
 					Type:       this.compressionCacheWriter.ItemType(),
 					Key:        this.compressionCacheWriter.Key(),
-					ExpiredAt:  expiredAt,
+					ExpiresAt:  expiredAt,
 					StaleAt:    expiredAt + int64(this.calculateStaleLife()),
 					HeaderSize: this.compressionCacheWriter.HeaderSize(),
 					BodySize:   this.compressionCacheWriter.BodySize(),

internal/nodes/http_writer_ext.go

@@ -11,3 +11,7 @@ import (
 func (this *HTTPWriter) canSendfile() (*os.File, bool) {
 	return nil, false
 }
+
+func (this *HTTPWriter) checkPlanBandwidth(n int) {
+	// stub
+}

internal/nodes/listener_http.go

@@ -3,6 +3,7 @@ package nodes
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/iwind/TeaGo/Tea"
 	"io"
@@ -52,6 +53,8 @@ func (this *HTTPListener) Serve() error {
 				atomic.AddInt64(&this.countActiveConnections, 1)
 			case http.StateClosed:
 				atomic.AddInt64(&this.countActiveConnections, -1)
+			default:
+				// do nothing
 			}
 		},
 		ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
@@ -74,7 +77,7 @@ func (this *HTTPListener) Serve() error {
 	// HTTP协议
 	if this.isHTTP {
 		err := this.httpServer.Serve(this.Listener)
-		if err != nil && err != http.ErrServerClosed {
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
 			return err
 		}
 	}
@@ -84,7 +87,7 @@ func (this *HTTPListener) Serve() error {
 		this.httpServer.TLSConfig = this.buildTLSConfig()
 
 		err := this.httpServer.ServeTLS(this.Listener, "", "")
-		if err != nil && err != http.ErrServerClosed {
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
 			return err
 		}
 	}
@@ -105,8 +108,13 @@ func (this *HTTPListener) Reload(group *serverconfigs.ServerAddressGroup) {
 	this.Reset()
 }
 
-// ServerHTTP 处理HTTP请求
+// ServeHTTPWithAddr 处理HTTP请求
 func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.Request) {
+	this.ServeHTTPWithAddr(rawWriter, rawReq, this.addr)
+}
+
+// ServeHTTPWithAddr 处理HTTP请求并指定服务地址
+func (this *HTTPListener) ServeHTTPWithAddr(rawWriter http.ResponseWriter, rawReq *http.Request, serverAddr string) {
 	if len(rawReq.Host) > 253 {
 		http.Error(rawWriter, "Host too long.", http.StatusBadRequest)
 		time.Sleep(1 * time.Second) // make connection slow down
@@ -115,7 +123,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 
 	var globalServerConfig = sharedNodeConfig.GlobalServerConfig
 	if globalServerConfig != nil && !globalServerConfig.HTTPAll.SupportsLowVersionHTTP && (rawReq.ProtoMajor < 1 /** 0.x **/ || (rawReq.ProtoMajor == 1 && rawReq.ProtoMinor == 0 /** 1.0 **/)) {
-		http.Error(rawWriter, rawReq.Proto+" request is not supported.", http.StatusBadRequest)
+		http.Error(rawWriter, rawReq.Proto+" request is not supported.", http.StatusHTTPVersionNotSupported)
 		time.Sleep(1 * time.Second) // make connection slow down
 		return
 	}
@@ -175,10 +183,12 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 	}
 
 	// 绑定连接
+	var clientConn ClientConnInterface
 	if server != nil && server.Id > 0 {
 		var requestConn = rawReq.Context().Value(HTTPConnContextKey)
 		if requestConn != nil {
-			clientConn, ok := requestConn.(ClientConnInterface)
+			var ok bool
+			clientConn, ok = requestConn.(ClientConnInterface)
 			if ok {
 				var goNext = clientConn.SetServerId(server.Id)
 				if !goNext {
@@ -211,7 +221,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 		ReqServer:  server,
 		ReqHost:    reqHost,
 		ServerName: serverName,
-		ServerAddr: this.addr,
+		ServerAddr: serverAddr,
 		IsHTTP:     this.isHTTP,
 		IsHTTPS:    this.isHTTPS,
 		IsHTTP3:    this.isHTTP3,
@@ -219,6 +229,14 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 		nodeConfig: sharedNodeConfig,
 	}
 	req.Do()
+
+	// fix hijacked connection state
+	if req.isHijacked && clientConn != nil && this.httpServer.ConnState != nil {
+		netConn, ok := clientConn.(net.Conn)
+		if ok {
+			this.httpServer.ConnState(netConn, http.StateClosed)
+		}
+	}
 }
 
 // 检查host是否为IP

internal/nodes/node.go

@@ -784,9 +784,13 @@ func (this *Node) listenSock() error {
 				var costSeconds = time.Since(before).Seconds()
 				var gcStats = &debug.GCStats{}
 				debug.ReadGCStats(gcStats)
+				var pauseMS float64
+				if len(gcStats.Pause) > 0 {
+					pauseMS = gcStats.Pause[0].Seconds() * 1000
+				}
 				_ = cmd.Reply(&gosock.Command{
 					Params: map[string]any{
-						"pauseMS": gcStats.PauseTotal.Seconds() * 1000,
+						"pauseMS": pauseMS,
 						"costMS":  costSeconds * 1000,
 					},
 				})

internal/nodes/node_tasks.go

@@ -102,6 +102,8 @@ func (this *Node) execTask(rpcClient *rpc.RPCClient, task *pb.NodeTask) error {
 		err = this.execNetworkSecurityPolicyChangedTask(rpcClient)
 	case "webPPolicyChanged":
 		err = this.execWebPPolicyChangedTask(rpcClient)
+	case "planChanged":
+		err = this.execPlanChangedTask(rpcClient)
 	default:
 		// 特殊任务
 		if strings.HasPrefix(task.Type, "ipListDeleted") { // 删除IP名单

internal/nodes/node_tasks_ext.go

@@ -34,3 +34,7 @@ func (this *Node) execNetworkSecurityPolicyChangedTask(rpcClient *rpc.RPCClient)
 	// stub
 	return nil
 }
+
+func (this *Node) execPlanChangedTask(rpcClient *rpc.RPCClient) error {
+	return nil
+}

internal/nodes/node_test.go

@@ -1,17 +1,26 @@
 package nodes
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
 )
 
 func TestNode_Start(t *testing.T) {
-	node := NewNode()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var node = NewNode()
 	node.Start()
 }
 
 func TestNode_Test(t *testing.T) {
-	node := NewNode()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var node = NewNode()
 	err := node.Test()
 	if err != nil {
 		t.Fatal(err)

internal/nodes/upgrade_manager_test.go

@@ -3,11 +3,16 @@
 package nodes
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
 )
 
 func TestUpgradeManager_install(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	err := NewUpgradeManager().install()
 	if err != nil {
 		t.Fatal(err)

internal/rpc/rpc_client.go

@@ -55,6 +55,7 @@ type RPCClient struct {
 	ClientAgentIPRPC       pb.ClientAgentIPServiceClient
 	AuthorityKeyRPC        pb.AuthorityKeyServiceClient
 	UpdatingServerListRPC  pb.UpdatingServerListServiceClient
+	PlanRPC                pb.PlanServiceClient
 }
 
 func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
@@ -91,6 +92,7 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
 	client.ClientAgentIPRPC = pb.NewClientAgentIPServiceClient(client)
 	client.AuthorityKeyRPC = pb.NewAuthorityKeyServiceClient(client)
 	client.UpdatingServerListRPC = pb.NewUpdatingServerListServiceClient(client)
+	client.PlanRPC = pb.NewPlanServiceClient(client)
 
 	err := client.init()
 	if err != nil {

internal/rpc/rpc_test.go

@@ -5,6 +5,7 @@ package rpc_test
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"sync"
@@ -13,6 +14,10 @@ import (
 )
 
 func TestRPCConcurrentCall(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	rpcClient, err := rpc.SharedRPC()
 	if err != nil {
 		t.Fatal(err)
@@ -43,6 +48,10 @@ func TestRPCConcurrentCall(t *testing.T) {
 }
 
 func TestRPC_Retry(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	rpcClient, err := rpc.SharedRPC()
 	if err != nil {
 		t.Fatal(err)

internal/stats/user_agent_parser.go

@@ -3,38 +3,51 @@
 package stats
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	syncutils "github.com/TeaOSLab/EdgeNode/internal/utils/sync"
 	"github.com/mssola/useragent"
 	"sync"
+	"time"
 )
 
 var SharedUserAgentParser = NewUserAgentParser()
 
+const userAgentShardingCount = 8
+
 // UserAgentParser UserAgent解析器
 type UserAgentParser struct {
-	parser *useragent.UserAgent
+	cacheMaps [userAgentShardingCount]map[uint64]UserAgentParserResult
+	pool      *sync.Pool
+	mu        *syncutils.RWMutex
 
-	cacheMap1     map[uint64]UserAgentParserResult
-	cacheMap2     map[uint64]UserAgentParserResult
 	maxCacheItems int
-
-	cacheCursor int
-	locker      sync.RWMutex
+	gcTicker      *time.Ticker
+	gcIndex       int
 }
 
+// NewUserAgentParser 获取新解析器
 func NewUserAgentParser() *UserAgentParser {
 	var parser = &UserAgentParser{
-		parser:      &useragent.UserAgent{},
-		cacheMap1:   map[uint64]UserAgentParserResult{},
-		cacheMap2:   map[uint64]UserAgentParserResult{},
-		cacheCursor: 0,
+		pool: &sync.Pool{
+			New: func() any {
+				return &useragent.UserAgent{}
+			},
+		},
+		cacheMaps: [userAgentShardingCount]map[uint64]UserAgentParserResult{},
+		mu:        syncutils.NewRWMutex(userAgentShardingCount),
+	}
+
+	for i := 0; i < userAgentShardingCount; i++ {
+		parser.cacheMaps[i] = map[uint64]UserAgentParserResult{}
 	}
 
 	parser.init()
 	return parser
 }
 
+// 初始化
 func (this *UserAgentParser) init() {
 	var maxCacheItems = 10_000
 	var systemMemory = utils.SystemMemoryGB()
@@ -46,8 +59,16 @@ func (this *UserAgentParser) init() {
 		maxCacheItems = 20_000
 	}
 	this.maxCacheItems = maxCacheItems
+
+	this.gcTicker = time.NewTicker(5 * time.Second)
+	goman.New(func() {
+		for range this.gcTicker.C {
+			this.GC()
+		}
+	})
 }
 
+// Parse 解析UserAgent
 func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResult) {
 	// 限制长度
 	if len(userAgent) == 0 || len(userAgent) > 256 {
@@ -55,28 +76,22 @@ func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResu
 	}
 
 	var userAgentKey = fnv.HashString(userAgent)
+	var shardingIndex = int(userAgentKey % userAgentShardingCount)
 
-	this.locker.RLock()
-	cacheResult, ok := this.cacheMap1[userAgentKey]
+	this.mu.RLock(shardingIndex)
+	cacheResult, ok := this.cacheMaps[shardingIndex][userAgentKey]
 	if ok {
-		this.locker.RUnlock()
+		this.mu.RUnlock(shardingIndex)
 		return cacheResult
 	}
+	this.mu.RUnlock(shardingIndex)
 
-	cacheResult, ok = this.cacheMap2[userAgentKey]
-	if ok {
-		this.locker.RUnlock()
-		return cacheResult
-	}
-	this.locker.RUnlock()
-
-	this.locker.Lock()
-	defer this.locker.Unlock()
-
-	this.parser.Parse(userAgent)
-	result.OS = this.parser.OSInfo()
-	result.BrowserName, result.BrowserVersion = this.parser.Browser()
-	result.IsMobile = this.parser.Mobile()
+	var parser = this.pool.Get().(*useragent.UserAgent)
+	parser.Parse(userAgent)
+	result.OS = parser.OSInfo()
+	result.BrowserName, result.BrowserVersion = parser.Browser()
+	result.IsMobile = parser.Mobile()
+	this.pool.Put(parser)
 
 	// 忽略特殊字符
 	if len(result.BrowserName) > 0 {
@@ -87,19 +102,45 @@ func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResu
 		}
 	}
 
-	if this.cacheCursor == 0 {
-		this.cacheMap1[userAgentKey] = result
-		if len(this.cacheMap1) >= this.maxCacheItems {
-			this.cacheCursor = 1
-			this.cacheMap2 = map[uint64]UserAgentParserResult{}
-		}
-	} else {
-		this.cacheMap2[userAgentKey] = result
-		if len(this.cacheMap2) >= this.maxCacheItems {
-			this.cacheCursor = 0
-			this.cacheMap1 = map[uint64]UserAgentParserResult{}
-		}
-	}
+	this.mu.Lock(shardingIndex)
+	this.cacheMaps[shardingIndex][userAgentKey] = result
+	this.mu.Unlock(shardingIndex)
 
 	return
 }
+
+// MaxCacheItems 读取能容纳的缓存最大数量
+func (this *UserAgentParser) MaxCacheItems() int {
+	return this.maxCacheItems
+}
+
+// Len 读取当前缓存数量
+func (this *UserAgentParser) Len() int {
+	var total = 0
+	for i := 0; i < userAgentShardingCount; i++ {
+		this.mu.RLock(i)
+		total += len(this.cacheMaps[i])
+		this.mu.RUnlock(i)
+	}
+	return total
+}
+
+// GC 回收多余的缓存
+func (this *UserAgentParser) GC() {
+	var total = this.Len()
+	if total > this.maxCacheItems {
+		for {
+			var shardingIndex = this.gcIndex
+
+			this.mu.Lock(shardingIndex)
+			total -= len(this.cacheMaps[shardingIndex])
+			this.cacheMaps[shardingIndex] = map[uint64]UserAgentParserResult{}
+			this.gcIndex = (this.gcIndex + 1) % userAgentShardingCount
+			this.mu.Unlock(shardingIndex)
+
+			if total <= this.maxCacheItems {
+				break
+			}
+		}
+	}
+}

internal/stats/user_agent_parser_test.go

@@ -1,17 +1,21 @@
 // Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 
-package stats
+package stats_test
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/stats"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/iwind/TeaGo/assert"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
 	"runtime"
 	"runtime/debug"
 	"testing"
+	"time"
 )
 
 func TestUserAgentParser_Parse(t *testing.T) {
-	var parser = NewUserAgentParser()
+	var parser = stats.NewUserAgentParser()
 	for i := 0; i < 4; i++ {
 		t.Log(parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/1"))
 		t.Log(parser.Parse("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"))
@@ -19,7 +23,7 @@ func TestUserAgentParser_Parse(t *testing.T) {
 }
 
 func TestUserAgentParser_Parse_Unknown(t *testing.T) {
-	var parser = NewUserAgentParser()
+	var parser = stats.NewUserAgentParser()
 	t.Log(parser.Parse("Mozilla/5.0 (Wind 10.0; WOW64; rv:49.0) Apple/537.36 (KHTML, like Gecko) Chr/88.0.4324.96 Sa/537.36 Test/1"))
 	t.Log(parser.Parse(""))
 }
@@ -28,10 +32,10 @@ func TestUserAgentParser_Memory(t *testing.T) {
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
 
-	var parser = NewUserAgentParser()
+	var parser = stats.NewUserAgentParser()
 
 	for i := 0; i < 1_000_000; i++ {
-		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
+		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
 	}
 
 	runtime.GC()
@@ -40,32 +44,76 @@ func TestUserAgentParser_Memory(t *testing.T) {
 	var stat2 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat2)
 
-	t.Log("max cache items:", parser.maxCacheItems)
-	t.Log("cache1:", len(parser.cacheMap1), "cache2:", len(parser.cacheMap2), "cache3:", (stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
+	t.Log("max cache items:", parser.MaxCacheItems())
+	t.Log("cache:", parser.Len(), "usage:", (stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
 }
 
-func BenchmarkUserAgentParser_Parse(b *testing.B) {
-	var parser = NewUserAgentParser()
-	for i := 0; i < b.N; i++ {
-		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
+func TestNewUserAgentParser_GC(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
 	}
-	b.Log(len(parser.cacheMap1), len(parser.cacheMap2))
-}
 
-func BenchmarkUserAgentParser_Parse2(b *testing.B) {
-	var parser = NewUserAgentParser()
-	for i := 0; i < b.N; i++ {
+	var parser = stats.NewUserAgentParser()
+
+	for i := 0; i < 1_000_000; i++ {
 		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
 	}
-	b.Log(len(parser.cacheMap1), len(parser.cacheMap2))
+
+	time.Sleep(60 * time.Second) // wait to gc
+	t.Log(parser.Len(), "cache items")
 }
 
-func BenchmarkUserAgentParser_Parse3(b *testing.B) {
-	var parser = NewUserAgentParser()
+func TestNewUserAgentParser_Mobile(t *testing.T) {
+	var a = assert.NewAssertion(t)
+	var parser = stats.NewUserAgentParser()
+	for _, userAgent := range []string{
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",
+		"Mozilla/5.0 (Linux; U; Android 2.2.1; en-us; Nexus One Build/FRG83) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+	} {
+		a.IsTrue(parser.Parse(userAgent).IsMobile)
+	}
+}
+
+func BenchmarkUserAgentParser_Parse_Many_LimitCPU(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var parser = stats.NewUserAgentParser()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
+		}
+	})
+	b.Log(parser.Len())
+}
+
+func BenchmarkUserAgentParser_Parse_Many(b *testing.B) {
+	var parser = stats.NewUserAgentParser()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
+		}
+	})
+	b.Log(parser.Len())
+}
+
+func BenchmarkUserAgentParser_Parse_Few_LimitCPU(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var parser = stats.NewUserAgentParser()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
 			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
 		}
 	})
-	b.Log(len(parser.cacheMap1), len(parser.cacheMap2))
+	b.Log(parser.Len())
+}
+
+func BenchmarkUserAgentParser_Parse_Few(b *testing.B) {
+	var parser = stats.NewUserAgentParser()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
+		}
+	})
+	b.Log(parser.Len())
 }

internal/ttlcache/cache_test.go

@@ -1,6 +1,7 @@
 package ttlcache
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
@@ -8,6 +9,7 @@ import (
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"runtime"
+	"runtime/debug"
 	"strconv"
 	"sync/atomic"
 	"testing"
@@ -48,16 +50,34 @@ func TestCache_Memory(t *testing.T) {
 	}
 
 	var cache = NewCache[int]()
+	var isReady bool
 
 	testutils.StartMemoryStats(t, func() {
+		if !isReady {
+			return
+		}
 		t.Log(cache.Count(), "items")
 	})
 
-	var count = 20_000_000
+	var count = 1_000_000
+	if utils.SystemMemoryGB() > 4 {
+		count = 20_000_000
+	}
 	for i := 0; i < count; i++ {
 		cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+int64(rands.Int(0, 300)))
 	}
 
+	func() {
+		var before = time.Now()
+		runtime.GC()
+		var costSeconds = time.Since(before).Seconds()
+		var stats = &debug.GCStats{}
+		debug.ReadGCStats(stats)
+		t.Log("GC pause:", stats.Pause[0].Seconds()*1000, "ms", "cost:", costSeconds*1000, "ms")
+	}()
+
+	isReady = true
+
 	t.Log(cache.Count())
 
 	time.Sleep(10 * time.Second)
@@ -105,6 +125,10 @@ func TestCache_IncreaseInt64(t *testing.T) {
 }
 
 func TestCache_Read(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	runtime.GOMAXPROCS(1)
 
 	var cache = NewCache[int](PiecesOption{Count: 32})

internal/utils/agents/manager_test.go

@@ -4,12 +4,17 @@ package agents_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/agents"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
 )
 
 func TestNewManager(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = agents.NewDB(Tea.Root + "/data/agents.db")
 	err := db.Init()
 	if err != nil {

internal/utils/agents/queue_test.go

@@ -4,6 +4,7 @@ package agents_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/agents"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
@@ -11,6 +12,10 @@ import (
 )
 
 func TestParseQueue_Process(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var queue = agents.NewQueue()
 	go queue.Start()
 	time.Sleep(1 * time.Second)
@@ -19,6 +24,10 @@ func TestParseQueue_Process(t *testing.T) {
 }
 
 func TestParseQueue_ParseIP(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var queue = agents.NewQueue()
 	for _, ip := range []string{
 		"192.168.1.100",

internal/utils/byte_pool_test.go

@@ -1,6 +1,8 @@
-package utils
+package utils_test
 
 import (
+	"bytes"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"runtime"
 	"sync"
 	"testing"
@@ -10,7 +12,7 @@ func TestBytePool_Memory(t *testing.T) {
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
 
-	var pool = NewBytePool(32 * 1024)
+	var pool = utils.NewBytePool(32 * 1024)
 	for i := 0; i < 20480; i++ {
 		pool.Put(make([]byte, 32*1024))
 	}
@@ -29,7 +31,7 @@ func TestBytePool_Memory(t *testing.T) {
 func BenchmarkBytePool_Get(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var pool = NewBytePool(1)
+	var pool = utils.NewBytePool(1)
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
@@ -42,7 +44,7 @@ func BenchmarkBytePool_Get(b *testing.B) {
 func BenchmarkBytePool_Get_Parallel(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var pool = NewBytePool(1024)
+	var pool = utils.NewBytePool(1024)
 	b.ResetTimer()
 
 	b.RunParallel(func(pb *testing.PB) {
@@ -72,3 +74,20 @@ func BenchmarkBytePool_Get_Sync(b *testing.B) {
 		}
 	})
 }
+
+func BenchmarkBytePool_Copy(b *testing.B) {
+	var data = bytes.Repeat([]byte{'A'}, 8<<10)
+
+	var pool = &sync.Pool{
+		New: func() any {
+			return make([]byte, 8<<10)
+		},
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var buf = pool.Get().([]byte)
+			copy(buf, data)
+			pool.Put(buf)
+		}
+	})
+}

internal/utils/clock/manager_test.go

@@ -4,9 +4,14 @@ package clock_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/clock"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 )
 
 func TestReadServer(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	t.Log(clock.NewClockManager().ReadServer("pool.ntp.org"))
 }

internal/utils/counters/counter.go

@@ -22,7 +22,7 @@ type SupportedUIntType interface {
 type Counter[T SupportedUIntType] struct {
 	countMaps uint64
 	locker    *syncutils.RWMutex
-	itemMaps  []map[uint64]*Item[T]
+	itemMaps  []map[uint64]Item[T]
 
 	gcTicker *time.Ticker
 	gcIndex  int
@@ -36,9 +36,9 @@ func NewCounter[T SupportedUIntType]() *Counter[T] {
 		count = 8
 	}
 
-	var itemMaps = []map[uint64]*Item[T]{}
+	var itemMaps = []map[uint64]Item[T]{}
 	for i := 0; i < count; i++ {
-		itemMaps = append(itemMaps, map[uint64]*Item[T]{})
+		itemMaps = append(itemMaps, map[uint64]Item[T]{})
 	}
 
 	var counter = &Counter[T]{
@@ -69,19 +69,22 @@ func (this *Counter[T]) WithGC() *Counter[T] {
 func (this *Counter[T]) Increase(key uint64, lifeSeconds int) T {
 	var index = int(key % this.countMaps)
 	this.locker.RLock(index)
-	var item = this.itemMaps[index][key]
+	var item = this.itemMaps[index][key] // item MUST NOT be pointer
 	this.locker.RUnlock(index)
-	if item == nil {
+	if !item.IsOk() {
 		// no need to care about duplication
 		// always insert new item even when itemMap is full
 		item = NewItem[T](lifeSeconds)
+		var result = item.Increase()
 		this.locker.Lock(index)
 		this.itemMaps[index][key] = item
 		this.locker.Unlock(index)
+		return result
 	}
 
 	this.locker.Lock(index)
 	var result = item.Increase()
+	this.itemMaps[index][key] = item // overwrite
 	this.locker.Unlock(index)
 	return result
 }
@@ -97,7 +100,7 @@ func (this *Counter[T]) Get(key uint64) T {
 	this.locker.RLock(index)
 	defer this.locker.RUnlock(index)
 	var item = this.itemMaps[index][key]
-	if item != nil {
+	if item.IsOk() {
 		return item.Sum()
 	}
 	return 0
@@ -115,7 +118,7 @@ func (this *Counter[T]) Reset(key uint64) {
 	var item = this.itemMaps[index][key]
 	this.locker.RUnlock(index)
 
-	if item != nil {
+	if item.IsOk() {
 		this.locker.Lock(index)
 		delete(this.itemMaps[index], key)
 		this.locker.Unlock(index)

internal/utils/counters/counter_test.go

@@ -101,7 +101,7 @@ func TestCounterMemory(t *testing.T) {
 		var costSeconds = time.Since(before).Seconds()
 		var stats = &debug.GCStats{}
 		debug.ReadGCStats(stats)
-		t.Log("GC pause:", stats.PauseTotal.Seconds()*1000, "ms", "cost:", costSeconds*1000, "ms")
+		t.Log("GC pause:", stats.Pause[0].Seconds()*1000, "ms", "cost:", costSeconds*1000, "ms")
 	}
 
 	gcPause()
@@ -113,12 +113,14 @@ func BenchmarkCounter_Increase(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
 	var counter = counters.NewCounter[uint32]()
+
+	b.ReportAllocs()
 	b.ResetTimer()
 
 	var i uint64
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			counter.Increase(atomic.AddUint64(&i, 1)%1e6, 20)
+			counter.Increase(atomic.AddUint64(&i, 1)%1_000_000, 20)
 		}
 	})
 
@@ -138,11 +140,12 @@ func BenchmarkCounter_IncreaseKey(b *testing.B) {
 	}()
 
 	b.ResetTimer()
+	b.ReportAllocs()
 
 	var i uint64
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			counter.IncreaseKey(types.String(atomic.AddUint64(&i, 1)%1e6), 20)
+			counter.IncreaseKey(types.String(atomic.AddUint64(&i, 1)%1_000_000), 20)
 		}
 	})
 

internal/utils/counters/item.go

@@ -16,7 +16,7 @@ type Item[T SupportedUIntType] struct {
 	spanSeconds    int64
 }
 
-func NewItem[T SupportedUIntType](lifeSeconds int) *Item[T] {
+func NewItem[T SupportedUIntType](lifeSeconds int) Item[T] {
 	if lifeSeconds <= 0 {
 		lifeSeconds = 60
 	}
@@ -27,7 +27,7 @@ func NewItem[T SupportedUIntType](lifeSeconds int) *Item[T] {
 		spanSeconds++
 	}
 
-	return &Item[T]{
+	return Item[T]{
 		lifeSeconds:    int64(lifeSeconds),
 		spanSeconds:    int64(spanSeconds),
 		lastUpdateTime: fasttime.Now().Unix(),
@@ -126,3 +126,7 @@ func (this *Item[T]) calculateSpanIndex(timestamp int64) int {
 	}
 	return index
 }
+
+func (this *Item[T]) IsOk() bool {
+	return this.lifeSeconds > 0
+}

internal/utils/fasttime/time_fast.go

@@ -36,6 +36,7 @@ type FastTime struct {
 	unixTimeMilliString string
 	ymd                 string
 	round5Hi            string
+	hour                int
 }
 
 func NewFastTime() *FastTime {
@@ -48,6 +49,7 @@ func NewFastTime() *FastTime {
 		unixTimeMilliString: types.String(rawTime.UnixMilli()),
 		ymd:                 timeutil.Format("Ymd", rawTime),
 		round5Hi:            timeutil.FormatTime("Hi", rawTime.Unix()/300*300),
+		hour:                rawTime.Hour(),
 	}
 }
 
@@ -91,3 +93,7 @@ func (this *FastTime) Round5Hi() string {
 func (this *FastTime) Format(layout string) string {
 	return timeutil.Format(layout, this.rawTime)
 }
+
+func (this *FastTime) Hour() int {
+	return this.hour
+}

internal/utils/fasttime/time_fast_test.go

@@ -38,6 +38,11 @@ func TestFastTime_Format(t *testing.T) {
 	t.Log(now.Format("Y-m-d H:i:s"))
 }
 
+func TestFastTime_Hour(t *testing.T) {
+	var now = fasttime.Now()
+	t.Log(now.Hour())
+}
+
 func BenchmarkNewFastTime(b *testing.B) {
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {

internal/utils/ip_test.go

@@ -42,12 +42,12 @@ func TestIsIPv4(t *testing.T) {
 func TestIsIPv6(t *testing.T) {
 	var a = assert.NewAssertion(t)
 	a.IsFalse(utils.IsIPv6("192.168.1.1"))
-	a.IsFloat32(utils.IsIPv6("0.0.0.0"))
+	a.IsFalse(utils.IsIPv6("0.0.0.0"))
 	a.IsFalse(utils.IsIPv6("192.168.1.256"))
 	a.IsFalse(utils.IsIPv6("192.168.1"))
 	a.IsTrue(utils.IsIPv6("::1"))
 	a.IsTrue(utils.IsIPv6("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
-	a.IsTrue(utils.IsIPv4("::ffff:192.168.0.1"))
+	a.IsFalse(utils.IsIPv4("::ffff:192.168.0.1"))
 	a.IsTrue(utils.IsIPv6("::ffff:192.168.0.1"))
 }
 

internal/utils/kvstore/db.go

@@ -0,0 +1,68 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+import (
+	"errors"
+	"sync"
+)
+
+type DB struct {
+	store *Store
+
+	name      string
+	namespace string
+	tableMap  map[string]TableInterface
+
+	locker sync.RWMutex
+}
+
+func NewDB(store *Store, dbName string) (*DB, error) {
+	if !IsValidName(dbName) {
+		return nil, errors.New("invalid database name '" + dbName + "'")
+	}
+
+	return &DB{
+		store:     store,
+		name:      dbName,
+		namespace: "$" + dbName + "$",
+		tableMap:  map[string]TableInterface{},
+	}, nil
+}
+
+func (this *DB) AddTable(table TableInterface) {
+	table.SetNamespace([]byte(this.Namespace() + table.Name() + "$"))
+	table.SetDB(this)
+
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	this.tableMap[table.Name()] = table
+}
+
+func (this *DB) Name() string {
+	return this.name
+}
+
+func (this *DB) Namespace() string {
+	return this.namespace
+}
+
+func (this *DB) Store() *Store {
+	return this.store
+}
+
+func (this *DB) Close() error {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	var lastErr error
+	for _, table := range this.tableMap {
+		err := table.Close()
+		if err != nil {
+			lastErr = err
+		}
+	}
+
+	return lastErr
+}

internal/utils/kvstore/db_test.go

@@ -0,0 +1,48 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
+	"github.com/cockroachdb/pebble"
+	"testing"
+)
+
+func TestNewDB(t *testing.T) {
+	store, err := kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = store.Close()
+	}()
+
+	_, err = store.NewDB("TEST_DB")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testingStore = store
+	testInspectDB(t)
+}
+
+func testInspectDB(t *testing.T) {
+	if testingStore == nil {
+		return
+	}
+	it, err := testingStore.RawDB().NewIter(&pebble.IterOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = it.Close()
+	}()
+
+	for it.First(); it.Valid(); it.Next() {
+		valueBytes, valueErr := it.ValueAndErr()
+		if valueErr != nil {
+			t.Fatal(valueErr)
+		}
+		t.Log(string(it.Key()), "=>", string(valueBytes))
+	}
+}

internal/utils/kvstore/errors.go

@@ -0,0 +1,18 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+import (
+	"errors"
+	"github.com/cockroachdb/pebble"
+)
+
+var ErrTableNotFound = errors.New("table not found")
+var ErrKeyTooLong = errors.New("too long key")
+
+func IsKeyNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	return errors.Is(err, pebble.ErrNotFound)
+}

internal/utils/kvstore/item.go

@@ -0,0 +1,9 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+type Item[T any] struct {
+	Key      string
+	Value    T
+	FieldKey []byte
+}

internal/utils/kvstore/iterator_options.go

@@ -0,0 +1,17 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+import "github.com/cockroachdb/pebble"
+
+type IteratorOptions struct {
+	LowerBound []byte
+	UpperBound []byte
+}
+
+func (this *IteratorOptions) RawOptions() *pebble.IterOptions {
+	return &pebble.IterOptions{
+		LowerBound: this.LowerBound,
+		UpperBound: this.UpperBound,
+	}
+}

internal/utils/kvstore/logger.go

@@ -0,0 +1,17 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+type Logger struct {
+}
+
+func NewLogger() *Logger {
+	return &Logger{}
+}
+
+func (this *Logger) Infof(format string, args ...any) {
+
+}
+func (this *Logger) Fatalf(format string, args ...any) {
+
+}

internal/utils/kvstore/options.go

@@ -0,0 +1,9 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+import "github.com/cockroachdb/pebble"
+
+var DefaultWriteOptions = &pebble.WriteOptions{
+	Sync: false,
+}

internal/utils/kvstore/query.go

@@ -0,0 +1,483 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+)
+
+type DataType = int
+
+const (
+	DataTypeKey   DataType = 1
+	DataTypeField DataType = 2
+)
+
+type QueryOperator int
+
+const (
+	QueryOperatorGt  QueryOperator = 1
+	QueryOperatorGte QueryOperator = 2
+	QueryOperatorLt  QueryOperator = 3
+	QueryOperatorLte QueryOperator = 4
+)
+
+type QueryOperatorInfo struct {
+	Operator QueryOperator
+	Value    any
+}
+
+type IteratorFunc[T any] func(tx *Tx[T], item Item[T]) (goNext bool, err error)
+
+type Query[T any] struct {
+	table *Table[T]
+	tx    *Tx[T]
+
+	dataType  int
+	offsetKey string
+	limit     int
+	prefix    string
+	reverse   bool
+	forUpdate bool
+
+	keysOnly bool
+
+	fieldName      string
+	fieldReverse   bool
+	fieldOperators []QueryOperatorInfo
+	fieldPrefix    string
+	fieldOffsetKey []byte
+}
+
+func NewQuery[T any]() *Query[T] {
+	return &Query[T]{
+		limit:    -1,
+		dataType: DataTypeKey,
+	}
+}
+
+func (this *Query[T]) SetTable(table *Table[T]) *Query[T] {
+	this.table = table
+	return this
+}
+
+func (this *Query[T]) SetTx(tx *Tx[T]) *Query[T] {
+	this.tx = tx
+	return this
+}
+
+func (this *Query[T]) ForKey() *Query[T] {
+	this.dataType = DataTypeKey
+	return this
+}
+
+func (this *Query[T]) ForField() *Query[T] {
+	this.dataType = DataTypeField
+	return this
+}
+
+func (this *Query[T]) Limit(limit int) *Query[T] {
+	this.limit = limit
+	return this
+}
+
+func (this *Query[T]) Offset(offsetKey string) *Query[T] {
+	this.offsetKey = offsetKey
+	return this
+}
+
+func (this *Query[T]) Prefix(prefix string) *Query[T] {
+	this.prefix = prefix
+	return this
+}
+
+func (this *Query[T]) Desc() *Query[T] {
+	this.reverse = true
+	return this
+}
+
+func (this *Query[T]) ForUpdate() *Query[T] {
+	this.forUpdate = true
+	return this
+}
+
+func (this *Query[T]) KeysOnly() *Query[T] {
+	this.keysOnly = true
+	return this
+}
+
+func (this *Query[T]) FieldAsc(fieldName string) *Query[T] {
+	this.fieldName = fieldName
+	this.fieldReverse = false
+	return this
+}
+
+func (this *Query[T]) FieldDesc(fieldName string) *Query[T] {
+	this.fieldName = fieldName
+	this.fieldReverse = true
+	return this
+}
+
+func (this *Query[T]) FieldPrefix(fieldName string, fieldPrefix string) *Query[T] {
+	this.fieldName = fieldName
+	this.fieldPrefix = fieldPrefix
+	return this
+}
+
+func (this *Query[T]) FieldOffset(fieldOffset []byte) *Query[T] {
+	this.fieldOffsetKey = fieldOffset
+	return this
+}
+
+//func (this *Query[T]) FieldLt(value any) *Query[T] {
+//	this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
+//		Operator: QueryOperatorLt,
+//		Value:    value,
+//	})
+//	return this
+//}
+//
+//func (this *Query[T]) FieldLte(value any) *Query[T] {
+//	this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
+//		Operator: QueryOperatorLte,
+//		Value:    value,
+//	})
+//	return this
+//}
+//
+//func (this *Query[T]) FieldGt(value any) *Query[T] {
+//	this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
+//		Operator: QueryOperatorGt,
+//		Value:    value,
+//	})
+//	return this
+//}
+//
+//func (this *Query[T]) FieldGte(value any) *Query[T] {
+//	this.fieldOperators = append(this.fieldOperators, QueryOperatorInfo{
+//		Operator: QueryOperatorGte,
+//		Value:    value,
+//	})
+//	return this
+//}
+
+func (this *Query[T]) FindAll(fn IteratorFunc[T]) (err error) {
+	defer func() {
+		var panicErr = recover()
+		if panicErr != nil {
+			resultErr, ok := panicErr.(error)
+			if ok {
+				err = fmt.Errorf("execute query failed: %w", resultErr)
+			}
+		}
+	}()
+
+	if this.tx != nil {
+		defer func() {
+			_ = this.tx.Close()
+		}()
+
+		var itErr error
+		if len(this.fieldName) == 0 {
+			itErr = this.iterateKeys(fn)
+		} else {
+			itErr = this.iterateFields(fn)
+		}
+		if itErr != nil {
+			return itErr
+		}
+		return this.tx.Commit()
+	}
+
+	if this.table != nil {
+		var txFn func(fn func(tx *Tx[T]) error) error
+		if this.forUpdate {
+			txFn = this.table.WriteTx
+		} else {
+			txFn = this.table.ReadTx
+		}
+
+		return txFn(func(tx *Tx[T]) error {
+			this.tx = tx
+
+			if len(this.fieldName) == 0 {
+				return this.iterateKeys(fn)
+			}
+			return this.iterateFields(fn)
+		})
+	}
+
+	return errors.New("current query require 'table' or 'tx'")
+}
+
+func (this *Query[T]) iterateKeys(fn IteratorFunc[T]) error {
+	if this.limit == 0 {
+		return nil
+	}
+
+	var opt = &IteratorOptions{}
+
+	var prefix []byte
+	switch this.dataType {
+	case DataTypeKey:
+		prefix = append(this.table.Namespace(), KeyPrefix...)
+	case DataTypeField:
+		prefix = append(this.table.Namespace(), FieldPrefix...)
+	default:
+		prefix = append(this.table.Namespace(), KeyPrefix...)
+	}
+
+	var prefixLen = len(prefix)
+
+	if len(this.prefix) > 0 {
+		prefix = append(prefix, this.prefix...)
+	}
+
+	var offsetKey []byte
+	if this.reverse {
+		if len(this.offsetKey) > 0 {
+			offsetKey = append(prefix, this.offsetKey...)
+		} else {
+			offsetKey = append(prefix, 0xFF)
+		}
+
+		opt.LowerBound = prefix
+		opt.UpperBound = offsetKey
+	} else {
+		if len(this.offsetKey) > 0 {
+			offsetKey = append(prefix, this.offsetKey...)
+		} else {
+			offsetKey = prefix
+		}
+		opt.LowerBound = offsetKey
+		opt.UpperBound = append(offsetKey, 0xFF)
+	}
+
+	var hasOffsetKey = len(this.offsetKey) > 0
+
+	it, itErr := this.tx.NewIterator(opt)
+	if itErr != nil {
+		return itErr
+	}
+	defer func() {
+		_ = it.Close()
+	}()
+
+	var count int
+
+	var itemFn = func() (goNext bool, err error) {
+		var keyBytes = it.Key()
+
+		// skip first offset key
+		if hasOffsetKey {
+			hasOffsetKey = false
+
+			if bytes.Equal(keyBytes, offsetKey) {
+				return true, nil
+			}
+		}
+
+		// call fn
+		var value T
+		if !this.keysOnly {
+			valueBytes, valueErr := it.ValueAndErr()
+			if valueErr != nil {
+				return false, valueErr
+			}
+			value, err = this.table.encoder.Decode(valueBytes)
+			if err != nil {
+				return false, err
+			}
+		}
+
+		goNext, callbackErr := fn(this.tx, Item[T]{
+			Key:   string(keyBytes[prefixLen:]),
+			Value: value,
+		})
+		if callbackErr != nil {
+			return false, callbackErr
+		}
+		if !goNext {
+			return false, nil
+		}
+
+		// limit
+		if this.limit > 0 {
+			count++
+
+			if count >= this.limit {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	}
+
+	if this.reverse {
+		for it.Last(); it.Valid(); it.Prev() {
+			goNext, itemErr := itemFn()
+			if itemErr != nil {
+				return itemErr
+			}
+			if !goNext {
+				break
+			}
+		}
+	} else {
+		for it.First(); it.Valid(); it.Next() {
+			goNext, itemErr := itemFn()
+			if itemErr != nil {
+				return itemErr
+			}
+			if !goNext {
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
+func (this *Query[T]) iterateFields(fn IteratorFunc[T]) error {
+	if this.limit == 0 {
+		return nil
+	}
+
+	var hasOffsetKey = len(this.offsetKey) > 0 || len(this.fieldOffsetKey) > 0
+
+	var opt = &IteratorOptions{}
+
+	var prefix = this.table.FieldKey(this.fieldName)
+	prefix = append(prefix, '$')
+
+	if len(this.fieldPrefix) > 0 {
+		prefix = append(prefix, this.fieldPrefix...)
+	}
+
+	var offsetKey []byte
+	if this.fieldReverse {
+		if len(this.fieldOffsetKey) > 0 {
+			offsetKey = this.fieldOffsetKey
+		} else if len(this.offsetKey) > 0 {
+			offsetKey = append(prefix, this.offsetKey...)
+		} else {
+			offsetKey = append(prefix, 0xFF)
+		}
+		opt.LowerBound = prefix
+		opt.UpperBound = offsetKey
+	} else {
+		if len(this.fieldOffsetKey) > 0 {
+			offsetKey = this.fieldOffsetKey
+		} else if len(this.offsetKey) > 0 {
+			offsetKey = append(prefix, this.offsetKey...)
+			offsetKey = append(offsetKey, 0)
+		} else {
+			offsetKey = prefix
+		}
+
+		opt.LowerBound = offsetKey
+		opt.UpperBound = append(prefix, 0xFF)
+	}
+
+	it, itErr := this.tx.NewIterator(opt)
+	if itErr != nil {
+		return itErr
+	}
+	defer func() {
+		_ = it.Close()
+	}()
+
+	var count int
+
+	var itemFn = func() (goNext bool, err error) {
+		var fieldKeyBytes = it.Key()
+
+		fieldValueBytes, keyBytes, decodeKeyErr := this.table.DecodeFieldKey(this.fieldName, fieldKeyBytes)
+		if decodeKeyErr != nil {
+			return false, decodeKeyErr
+		}
+
+		// skip first offset key
+		if hasOffsetKey {
+			hasOffsetKey = false
+
+			if (len(this.fieldOffsetKey) > 0 && bytes.Equal(fieldKeyBytes, this.fieldOffsetKey)) ||
+				bytes.Equal(fieldValueBytes, []byte(this.offsetKey)) {
+				return true, nil
+			}
+		}
+
+		// 执行operators
+		if len(this.fieldOperators) > 0 {
+			if !this.matchOperators(fieldValueBytes) {
+				return true, nil
+			}
+		}
+
+		var resultItem = Item[T]{
+			Key:      string(keyBytes),
+			FieldKey: fieldKeyBytes,
+		}
+		if !this.keysOnly {
+			value, getErr := this.table.getWithKeyBytes(this.tx, this.table.FullKeyBytes(keyBytes))
+			if getErr != nil {
+				if IsKeyNotFound(getErr) {
+					return true, nil
+				}
+				return false, getErr
+			}
+
+			resultItem.Value = value
+		}
+
+		goNext, err = fn(this.tx, resultItem)
+		if err != nil {
+			return
+		}
+		if !goNext {
+			return false, nil
+		}
+
+		// limit
+		if this.limit > 0 {
+			count++
+
+			if count >= this.limit {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	}
+
+	if this.reverse {
+		for it.Last(); it.Valid(); it.Prev() {
+			goNext, itemErr := itemFn()
+			if itemErr != nil {
+				return itemErr
+			}
+			if !goNext {
+				break
+			}
+		}
+	} else {
+		for it.First(); it.Valid(); it.Next() {
+			goNext, itemErr := itemFn()
+			if itemErr != nil {
+				return itemErr
+			}
+			if !goNext {
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
+func (this *Query[T]) matchOperators(fieldValueBytes []byte) bool {
+	// TODO
+	return true
+}

internal/utils/kvstore/query_test.go

@@ -0,0 +1,276 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore_test
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
+	"github.com/iwind/TeaGo/assert"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func TestQuery_FindAll(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	var before = time.Now()
+	defer func() {
+		t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
+	}()
+
+	err := table.
+		Query().
+		Limit(10).
+		//Offset("a1000").
+		//Desc().
+		FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+			t.Log("key:", item.Key, "value:", item.Value)
+
+			return true, nil
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestQuery_FindAll_Break(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	var before = time.Now()
+	defer func() {
+		t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
+	}()
+
+	var count int
+	err := table.
+		Query().
+		FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+			t.Log("key:", item.Key, "value:", item.Value)
+			count++
+
+			if count > 2 {
+				// break test
+				_ = table.DB().Store().Close()
+			}
+
+			return count < 3, nil
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestQuery_FindAll_Break_Closed(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	var a = assert.NewAssertion(t)
+
+	var before = time.Now()
+	defer func() {
+		t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
+	}()
+
+	var count int
+	err := table.
+		Query().
+		FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+			t.Log("key:", item.Key, "value:", item.Value)
+			count++
+
+			if count > 2 {
+				// break test
+				_ = table.DB().Store().Close()
+			}
+
+			return count < 3, nil
+		})
+	t.Log("expected error:", err)
+	a.IsTrue(err != nil)
+}
+
+func TestQuery_FindAll_Desc(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	err := table.Query().
+		Desc().
+		Limit(10).
+		FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+			t.Log("key:", item.Key, "value:", item.Value)
+			return true, nil
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestQuery_FindAll_Offset(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	{
+		t.Log("=== forward ===")
+		err := table.Query().
+			Offset("a3").
+			Limit(10).
+			FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+				t.Log("key:", item.Key, "value:", item.Value)
+				return true, nil
+			})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	{
+		t.Log("=== backward ===")
+		err := table.Query().
+			Desc().
+			Offset("a3").
+			Limit(10).
+			//KeyOnly().
+			FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+				t.Log("key:", item.Key, "value:", item.Value)
+				return true, nil
+			})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestQuery_FindAll_Count(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	var count int
+
+	var before = time.Now()
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", costSeconds*1000, "ms", "qps:", fmt.Sprintf("%.2fM/s", float64(count)/costSeconds/1_000_000))
+	}()
+
+	err := table.
+		Query().
+		KeysOnly().
+		FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+			count++
+			return true, nil
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("count:", count)
+}
+
+func TestQuery_FindAll_Field(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	var before = time.Now()
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", costSeconds*1000, "ms", "qps:", int(1/costSeconds))
+	}()
+
+	var lastFieldKey []byte
+
+	t.Log("=======")
+	{
+		err := table.
+			Query().
+			FieldAsc("expiresAt").
+			//KeysOnly().
+			//FieldLt(1710848959).
+			Limit(3).
+			FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+				t.Log(item.Key, "=>", item.Value)
+				lastFieldKey = item.FieldKey
+				return true, nil
+			})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+	}
+
+	t.Log("=======")
+	{
+		err := table.
+			Query().
+			FieldAsc("expiresAt").
+			//KeysOnly().
+			//FieldLt(1710848959).
+			FieldOffset(lastFieldKey).
+			Limit(3).
+			FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+				t.Log(item.Key, "=>", item.Value)
+				lastFieldKey = item.FieldKey
+				return true, nil
+			})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestQuery_FindAll_Field_Many(t *testing.T) {
+	var table = testOpenStoreTable[*testCachedItem](t, "cache_items", &testCacheItemEncoder[*testCachedItem]{})
+
+	var before = time.Now()
+	defer func() {
+		var costSeconds = time.Since(before).Seconds()
+		t.Log("cost:", costSeconds*1000, "ms", "qps:", int(1/costSeconds))
+	}()
+
+	err := table.
+		Query().
+		FieldAsc("expiresAt").
+		KeysOnly().
+		Limit(1000).
+		FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+			t.Log(item.Key, "=>", item.Value)
+			return true, nil
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func BenchmarkQuery_FindAll(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	store, err := kvstore.OpenStore("test")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer func() {
+		_ = store.Close()
+	}()
+
+	db, err := store.NewDB("db1")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	table, err := kvstore.NewTable[*testCachedItem]("cache_items", &testCacheItemEncoder[*testCachedItem]{})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	db.AddTable(table)
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			err = table.Query().
+				//Limit(100).
+				FindAll(func(tx *kvstore.Tx[*testCachedItem], item kvstore.Item[*testCachedItem]) (goNext bool, err error) {
+					return true, nil
+				})
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}

internal/utils/kvstore/store.go

@@ -0,0 +1,169 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+import (
+	"errors"
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/events"
+	"github.com/cockroachdb/pebble"
+	"github.com/iwind/TeaGo/Tea"
+	"io"
+	"os"
+	"strings"
+	"sync"
+)
+
+const StoreSuffix = ".store"
+
+type Store struct {
+	name string
+
+	path  string
+	rawDB *pebble.DB
+
+	isClosed bool
+
+	dbs []*DB
+
+	mu sync.Mutex
+}
+
+// NewStore create store with name
+func NewStore(storeName string) (*Store, error) {
+	if !IsValidName(storeName) {
+		return nil, errors.New("invalid store name '" + storeName + "'")
+	}
+
+	var root = Tea.Root + "/data/stores"
+	_, err := os.Stat(root)
+	if err != nil && os.IsNotExist(err) {
+		_ = os.MkdirAll(root, 0777)
+	}
+
+	return &Store{
+		name: storeName,
+		path: Tea.Root + "/data/stores/" + storeName + StoreSuffix,
+	}, nil
+}
+
+func OpenStore(storeName string) (*Store, error) {
+	store, err := NewStore(storeName)
+	if err != nil {
+		return nil, err
+	}
+	err = store.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	return store, nil
+}
+
+func OpenStoreDir(dir string, storeName string) (*Store, error) {
+	if !IsValidName(storeName) {
+		return nil, errors.New("invalid store name '" + storeName + "'")
+	}
+
+	_, err := os.Stat(dir)
+	if err != nil && os.IsNotExist(err) {
+		_ = os.MkdirAll(dir, 0777)
+	}
+
+	dir = strings.TrimSuffix(dir, "/")
+
+	var store = &Store{
+		name: storeName,
+		path: dir + "/" + storeName + StoreSuffix,
+	}
+
+	err = store.Open()
+	if err != nil {
+		return nil, err
+	}
+	return store, nil
+}
+
+func (this *Store) Open() error {
+	var opt = &pebble.Options{
+		Logger: NewLogger(),
+	}
+
+	// TODO 需要修改 BytesPerSync 和 WALBytesPerSync 等等默认参数
+
+	rawDB, err := pebble.Open(this.path, opt)
+	if err != nil {
+		return err
+	}
+	this.rawDB = rawDB
+
+	// events
+	events.OnKey(events.EventQuit, fmt.Sprintf("kvstore_%p", this), func() {
+		_ = this.Close()
+	})
+	events.OnKey(events.EventTerminated, fmt.Sprintf("kvstore_%p", this), func() {
+		_ = this.Close()
+	})
+
+	return nil
+}
+
+func (this *Store) Set(keyBytes []byte, valueBytes []byte) error {
+	return this.rawDB.Set(keyBytes, valueBytes, DefaultWriteOptions)
+}
+
+func (this *Store) Get(keyBytes []byte) (valueBytes []byte, closer io.Closer, err error) {
+	return this.rawDB.Get(keyBytes)
+}
+
+func (this *Store) Delete(keyBytes []byte) error {
+	return this.rawDB.Delete(keyBytes, DefaultWriteOptions)
+}
+
+func (this *Store) NewDB(dbName string) (*DB, error) {
+	db, err := NewDB(this, dbName)
+	if err != nil {
+		return nil, err
+	}
+
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	this.dbs = append(this.dbs, db)
+	return db, nil
+}
+
+func (this *Store) RawDB() *pebble.DB {
+	return this.rawDB
+}
+
+func (this *Store) Close() error {
+	if this.isClosed {
+		return nil
+	}
+
+	this.mu.Lock()
+	var lastErr error
+	for _, db := range this.dbs {
+		err := db.Close()
+		if err != nil {
+			lastErr = err
+		}
+	}
+
+	this.mu.Unlock()
+
+	if this.rawDB != nil {
+		this.isClosed = true
+		err := this.rawDB.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return lastErr
+}
+
+func (this *Store) IsClosed() bool {
+	return this.isClosed
+}

internal/utils/kvstore/store_test.go

@@ -0,0 +1,162 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
+	"github.com/cockroachdb/pebble"
+	"github.com/iwind/TeaGo/Tea"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"testing"
+	"time"
+)
+
+func TestMain(m *testing.M) {
+	m.Run()
+
+	if testingStore != nil {
+		_ = testingStore.Close()
+	}
+}
+
+func TestStore_Open(t *testing.T) {
+	store, err := kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = store.Close()
+	}()
+
+	t.Log("opened")
+}
+
+func TestStore_RawDB(t *testing.T) {
+	store, err := kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = store.Close()
+	}()
+
+	err = store.RawDB().Set([]byte("hello"), []byte("world"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestOpenStoreDir(t *testing.T) {
+	store, err := kvstore.OpenStoreDir(Tea.Root+"/data/stores", "test3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = store.Close()
+	}()
+
+	t.Log("opened")
+
+	_ = store
+}
+
+func TestStore_CloseTwice(t *testing.T) {
+	store, err := kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		for i := 0; i < 3; i++ {
+			err = store.Close()
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}()
+}
+
+func TestStore_Count(t *testing.T) {
+	testCountStore(t)
+}
+
+var testingStore *kvstore.Store
+
+func testOpenStore(t *testing.T) *kvstore.DB {
+	var err error
+	testingStore, err = kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db, err := testingStore.NewDB("db1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return db
+}
+
+func testOpenStoreTable[T any](t *testing.T, tableName string, encoder kvstore.ValueEncoder[T]) *kvstore.Table[T] {
+	var err error
+
+	var before = time.Now()
+	testingStore, err = kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("store open cost:", time.Since(before).Seconds()*1000, "ms")
+
+	db, err := testingStore.NewDB("db1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	table, err := kvstore.NewTable[T](tableName, encoder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	db.AddTable(table)
+
+	return table
+}
+
+func testOpenStoreTableForBenchmark[T any](t *testing.B, tableName string, encoder kvstore.ValueEncoder[T]) *kvstore.Table[T] {
+	var err error
+	testingStore, err = kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db, err := testingStore.NewDB("db1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	table, err := kvstore.NewTable[T](tableName, encoder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	db.AddTable(table)
+
+	return table
+}
+
+func testCountStore(t *testing.T) {
+	var err error
+	testingStore, err = kvstore.OpenStore("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var count int
+	it, err := testingStore.RawDB().NewIter(&pebble.IterOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = it.Close()
+	}()
+	for it.First(); it.Valid(); it.Next() {
+		count++
+	}
+	t.Log("count:", count)
+}

internal/utils/kvstore/table.go

@@ -0,0 +1,383 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"github.com/cockroachdb/pebble"
+	"github.com/iwind/TeaGo/types"
+	"sync"
+)
+
+const (
+	KeyPrefix    = "K$"
+	KeyMaxLength = 8 << 10
+
+	FieldPrefix = "F$"
+
+	MaxBatchKeys = 8 << 10 // TODO not implemented
+)
+
+type Table[T any] struct {
+	name         string
+	rawNamespace []byte
+	db           *DB
+	encoder      ValueEncoder[T]
+	fieldNames   []string
+
+	mu *sync.RWMutex
+}
+
+func NewTable[T any](tableName string, encoder ValueEncoder[T]) (*Table[T], error) {
+	if !IsValidName(tableName) {
+		return nil, errors.New("invalid table name '" + tableName + "'")
+	}
+
+	return &Table[T]{
+		name:    tableName,
+		encoder: encoder,
+		mu:      &sync.RWMutex{},
+	}, nil
+}
+
+func (this *Table[T]) Name() string {
+	return this.name
+}
+
+func (this *Table[T]) Namespace() []byte {
+	var dest = make([]byte, len(this.rawNamespace))
+	copy(dest, this.rawNamespace)
+	return dest
+}
+
+func (this *Table[T]) SetNamespace(namespace []byte) {
+	this.rawNamespace = namespace
+}
+
+func (this *Table[T]) SetDB(db *DB) {
+	this.db = db
+}
+
+func (this *Table[T]) DB() *DB {
+	return this.db
+}
+
+func (this *Table[T]) Set(key string, value T) error {
+	if len(key) > KeyMaxLength {
+		return ErrKeyTooLong
+	}
+
+	valueBytes, err := this.encoder.Encode(value)
+	if err != nil {
+		return err
+	}
+
+	return this.WriteTx(func(tx *Tx[T]) error {
+		return this.set(tx, key, valueBytes, value)
+	})
+}
+
+// ComposeFieldKey compose field key
+// $Namespace$FieldName$FieldValueSeparatorKeyValueFieldLength[2]
+func (this *Table[T]) ComposeFieldKey(keyBytes []byte, fieldName string, fieldValueBytes []byte) []byte {
+	// TODO use 'make()' and 'copy()' to pre-alloc memory space
+	var b = make([]byte, 2)
+	binary.BigEndian.PutUint16(b, uint16(len(fieldValueBytes)))
+	var fieldKey = append(this.FieldKey(fieldName), '$') // namespace
+	fieldKey = append(fieldKey, fieldValueBytes...)      // field value
+	fieldKey = append(fieldKey, 0, 0)                    // separator
+	fieldKey = append(fieldKey, keyBytes...)             // key value
+	fieldKey = append(fieldKey, b...)                    // field value length
+	return fieldKey
+}
+
+func (this *Table[T]) Exist(key string) (found bool, err error) {
+	_, closer, err := this.db.store.rawDB.Get(this.FullKey(key))
+	if err != nil {
+		if IsKeyNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	defer func() {
+		_ = closer.Close()
+	}()
+
+	return true, nil
+}
+
+func (this *Table[T]) Get(key string) (value T, err error) {
+	err = this.ReadTx(func(tx *Tx[T]) error {
+		resultValue, getErr := this.get(tx, key)
+		if getErr == nil {
+			value = resultValue
+		}
+		return getErr
+	})
+
+	return
+}
+
+func (this *Table[T]) Delete(key ...string) error {
+	if len(key) == 0 {
+		return nil
+	}
+
+	return this.WriteTx(func(tx *Tx[T]) error {
+		return this.deleteKeys(tx, key...)
+	})
+}
+
+func (this *Table[T]) ReadTx(fn func(tx *Tx[T]) error) error {
+	var tx = NewTx[T](this, true)
+	defer func() {
+		_ = tx.Close()
+	}()
+
+	err := fn(tx)
+	if err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (this *Table[T]) WriteTx(fn func(tx *Tx[T]) error) error {
+	var tx = NewTx[T](this, false)
+	defer func() {
+		_ = tx.Close()
+	}()
+
+	err := fn(tx)
+	if err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (this *Table[T]) Truncate() error {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	return this.db.store.rawDB.DeleteRange(this.Namespace(), append(this.Namespace(), 0xFF), DefaultWriteOptions)
+}
+
+func (this *Table[T]) Query() *Query[T] {
+	var query = NewQuery[T]()
+	query.SetTable(this)
+	return query
+}
+
+func (this *Table[T]) Count() (int64, error) {
+	var count int64
+
+	var begin = this.FullKeyBytes(nil)
+	it, err := this.db.store.rawDB.NewIter(&pebble.IterOptions{
+		LowerBound: begin,
+		UpperBound: append(begin, 0xFF),
+	})
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		_ = it.Close()
+	}()
+
+	for it.First(); it.Valid(); it.Next() {
+		count++
+	}
+
+	return count, err
+}
+
+func (this *Table[T]) FullKey(realKey string) []byte {
+	return append(this.Namespace(), KeyPrefix+realKey...)
+}
+
+func (this *Table[T]) FullKeyBytes(realKeyBytes []byte) []byte {
+	var k = append(this.Namespace(), KeyPrefix...)
+	k = append(k, realKeyBytes...)
+	return k
+}
+
+func (this *Table[T]) FieldKey(fieldName string) []byte {
+	var data = append(this.Namespace(), FieldPrefix...)
+	data = append(data, fieldName...)
+	return data
+}
+
+func (this *Table[T]) DecodeFieldKey(fieldName string, fieldKey []byte) (fieldValue []byte, key []byte, err error) {
+	var l = len(fieldKey)
+	var baseLen = len(this.FieldKey(fieldName)) + 1 /** $ **/ + 2 /** separator length **/ + 2 /** field length **/
+	if l < baseLen {
+		err = errors.New("invalid field key")
+		return
+	}
+
+	var fieldValueLen = binary.BigEndian.Uint16(fieldKey[l-2:])
+	var data = fieldKey[baseLen-4 : l-2]
+
+	fieldValue = data[:fieldValueLen]
+	key = data[fieldValueLen+2: /** separator length **/]
+
+	return
+}
+
+func (this *Table[T]) Close() error {
+	return nil
+}
+
+func (this *Table[T]) deleteKeys(tx *Tx[T], key ...string) error {
+	var batch = tx.batch
+
+	for _, singleKey := range key {
+		var keyErr = func(singleKey string) error {
+			var keyBytes = this.FullKey(singleKey)
+
+			// delete field values
+			if len(this.fieldNames) > 0 {
+				valueBytes, closer, getErr := batch.Get(keyBytes)
+				if getErr != nil {
+					if IsKeyNotFound(getErr) {
+						return nil
+					}
+					return getErr
+				}
+				defer func() {
+					_ = closer.Close()
+				}()
+
+				value, decodeErr := this.encoder.Decode(valueBytes)
+				if decodeErr != nil {
+					return decodeErr
+				}
+
+				for _, fieldName := range this.fieldNames {
+					fieldValueBytes, fieldErr := this.encoder.EncodeField(value, fieldName)
+					if fieldErr != nil {
+						return fieldErr
+					}
+
+					deleteKeyErr := batch.Delete(this.ComposeFieldKey([]byte(singleKey), fieldName, fieldValueBytes), DefaultWriteOptions)
+					if deleteKeyErr != nil {
+						return deleteKeyErr
+					}
+				}
+			}
+
+			err := batch.Delete(keyBytes, DefaultWriteOptions)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}(singleKey)
+		if keyErr != nil {
+			return keyErr
+		}
+	}
+
+	return nil
+}
+
+func (this *Table[T]) set(tx *Tx[T], key string, valueBytes []byte, value T) error {
+	var keyBytes = this.FullKey(key)
+
+	var batch = tx.batch
+
+	// read old value
+	var oldValue T
+	var oldFound bool
+	var countFields = len(this.fieldNames)
+	if countFields > 0 {
+		oldValueBytes, closer, getErr := batch.Get(keyBytes)
+		if getErr != nil {
+			if !IsKeyNotFound(getErr) {
+				return getErr
+			}
+		} else {
+			defer func() {
+				_ = closer.Close()
+			}()
+
+			var decodeErr error
+			oldValue, decodeErr = this.encoder.Decode(oldValueBytes)
+			if decodeErr != nil {
+				return decodeErr
+			}
+			oldFound = true
+		}
+	}
+
+	setErr := batch.Set(keyBytes, valueBytes, DefaultWriteOptions)
+	if setErr != nil {
+		return setErr
+	}
+
+	// process fields
+	if countFields > 0 {
+		// add new field keys
+		for _, fieldName := range this.fieldNames {
+			// 把EncodeField放在TX里，是为了节约内存
+			fieldValueBytes, fieldErr := this.encoder.EncodeField(value, fieldName)
+			if fieldErr != nil {
+				return fieldErr
+			}
+
+			if len(fieldValueBytes) > 8<<10 {
+				return errors.New("field value too long: " + types.String(len(fieldValueBytes)))
+			}
+
+			var newFieldKeyBytes = this.ComposeFieldKey([]byte(key), fieldName, fieldValueBytes)
+
+			// delete old field key
+			if oldFound {
+				oldFieldValueBytes, oldFieldErr := this.encoder.EncodeField(oldValue, fieldName)
+				if oldFieldErr != nil {
+					return oldFieldErr
+				}
+				var oldFieldKeyBytes = this.ComposeFieldKey([]byte(key), fieldName, oldFieldValueBytes)
+				if bytes.Equal(oldFieldKeyBytes, newFieldKeyBytes) {
+					// skip the field
+					continue
+				}
+				deleteFieldErr := batch.Delete(oldFieldKeyBytes, DefaultWriteOptions)
+				if deleteFieldErr != nil {
+					return deleteFieldErr
+				}
+			}
+
+			// set new field key
+			setFieldErr := batch.Set(newFieldKeyBytes, nil, DefaultWriteOptions)
+			if setFieldErr != nil {
+				return setFieldErr
+			}
+		}
+	}
+
+	return nil
+}
+
+func (this *Table[T]) get(tx *Tx[T], key string) (value T, err error) {
+	return this.getWithKeyBytes(tx, this.FullKey(key))
+}
+
+func (this *Table[T]) getWithKeyBytes(tx *Tx[T], keyBytes []byte) (value T, err error) {
+	valueBytes, closer, err := tx.batch.Get(keyBytes)
+	if err != nil {
+		return value, err
+	}
+	defer func() {
+		_ = closer.Close()
+	}()
+
+	resultValue, decodeErr := this.encoder.Decode(valueBytes)
+	if decodeErr != nil {
+		return value, decodeErr
+	}
+	value = resultValue
+	return
+}

internal/utils/kvstore/table_counter.go

@@ -0,0 +1,33 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package kvstore
+
+type CounterTable[T int64 | uint64] struct {
+	*Table[T]
+}
+
+func NewCounterTable[T int64 | uint64](name string) (*CounterTable[T], error) {
+	table, err := NewTable[T](name, NewIntValueEncoder[T]())
+	if err != nil {
+		return nil, err
+	}
+
+	return &CounterTable[T]{
+		Table: table,
+	}, nil
+}
+
+func (this *CounterTable[T]) Increase(key string, delta T) (newValue T, err error) {
+	err = this.Table.WriteTx(func(tx *Tx[T]) error {
+		value, getErr := tx.Get(key)
+		if getErr != nil {
+			if !IsKeyNotFound(getErr) {
+				return getErr
+			}
+		}
+
+		newValue = value + delta
+		return tx.Set(key, newValue)
+	})
+	return
+}